Prepare new PyPI release

set the default value of beta=1

.travis.yml

@@ -49,9 +49,9 @@ install:
 
   # install TensorFlow
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl;
+      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0-cp27-none-linux_x86_64.whl;
     elif [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl;
+      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0-cp34-cp34m-linux_x86_64.whl;
     fi
 # command to run tests
 script:

README.md

@@ -8,7 +8,7 @@
 
 ## You have just found Keras.
 
-Keras is a minimalist, highly modular neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
+Keras is a high-level neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
 
 Use Keras if you need a deep learning library that:
 
@@ -149,7 +149,10 @@ By default, Keras will use TensorFlow as its tensor manipulation library. [Follo
 
 ## Support
 
-You can ask questions and join the development discussion on the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+You can ask questions and join the development discussion:
+
+- On the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+- On the [Keras Gitter channel](https://gitter.im/Keras-io/Lobby).
 
 You can also post bug reports and feature requests in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
 

docs/autogen.py

@@ -40,6 +40,7 @@ Index
     Sequence preprocessing
 
 Objectives
+Metrics
 Optimizers
 Activations
 Callbacks
@@ -79,10 +80,15 @@ from keras import callbacks
 from keras import models
 from keras.engine import topology
 from keras import objectives
+from keras import metrics
 from keras import backend
 from keras import constraints
 from keras import activations
 from keras import regularizers
+from keras.utils import data_utils
+from keras.utils import io_utils
+from keras.utils import layer_utils
+from keras.utils import np_utils
 
 
 EXCLUDE = {
@@ -133,6 +139,7 @@ PAGES = [
             core.Dense,
             core.Activation,
             core.Dropout,
+            core.SpatialDropout1D,
             core.SpatialDropout2D,
             core.SpatialDropout3D,
             core.Flatten,
@@ -158,6 +165,9 @@ PAGES = [
             convolutional.SeparableConvolution2D,
             convolutional.Deconvolution2D,
             convolutional.Convolution3D,
+            convolutional.Cropping1D,
+            convolutional.Cropping2D,
+            convolutional.Cropping3D,
             convolutional.UpSampling1D,
             convolutional.UpSampling2D,
             convolutional.UpSampling3D,
@@ -221,7 +231,10 @@ PAGES = [
         'page': 'layers/wrappers.md',
         'all_module_classes': [wrappers],
     },
-
+    {
+        'page': 'metrics.md',
+        'all_module_functions': [metrics],
+    },
     {
         'page': 'optimizers.md',
         'all_module_classes': [optimizers],
@@ -234,6 +247,28 @@ PAGES = [
         'page': 'backend.md',
         'all_module_functions': [backend],
     },
+    {
+        'page': 'utils/data_utils.md',
+        'functions': [
+            data_utils.get_file,
+        ]
+    },
+    {
+        'page': 'utils/io_utils.md',
+        'classes': [
+            io_utils.HDF5Matrix
+        ],
+    },
+    {
+        'page': 'utils/layer_utils.md',
+        'functions': [
+            layer_utils.layer_from_config,
+        ]
+    },
+    {
+        'page': 'utils/np_utils.md',
+        'all_module_functions': [np_utils]
+    },
 ]
 
 ROOT = 'http://keras.io/'

docs/mkdocs.yml

@@ -38,6 +38,7 @@ pages:
   - Text Preprocessing: preprocessing/text.md
   - Image Preprocessing: preprocessing/image.md
 - Objectives: objectives.md
+- Metrics: metrics.md
 - Optimizers: optimizers.md
 - Activations: activations.md
 - Callbacks: callbacks.md
@@ -49,7 +50,11 @@ pages:
 - Constraints: constraints.md
 - Visualization: visualization.md
 - Scikit-learn API: scikit-learn-api.md
-
+- Utils:
+  - Data Utils: utils/data_utils.md
+  - I/O Utils: utils/io_utils.md
+  - Layer Utils: utils/layer_utils.md
+  - Numpy Utils: utils/np_utils.md
 
 
 

docs/templates/applications.md

@@ -7,18 +7,25 @@ Weights are downloaded automatically when instantiating a model. They are stored
 
 ## Available models
 
-Models for image classification with weights trained on ImageNet:
+### Models for image classification with weights trained on ImageNet:
 
+- [Xception](#xception)
 - [VGG16](#vgg16)
 - [VGG19](#vgg19)
 - [ResNet50](#resnet50)
 - [InceptionV3](#inceptionv3)
 
-All of these architectures are compatible with both TensorFlow and Theano, and upon instantiation the models will be built according to the image dimension ordering set in your Keras configuration file at `~/.keras/keras.json`. For instance, if you have set `image_dim_ordering=tf`, then any model loaded from this repository will get built according to the TensorFlow dimension ordering convention, "Width-Height-Depth".
+All of these architectures (except Xception) are compatible with both TensorFlow and Theano, and upon instantiation the models will be built according to the image dimension ordering set in your Keras configuration file at `~/.keras/keras.json`. For instance, if you have set `image_dim_ordering=tf`, then any model loaded from this repository will get built according to the TensorFlow dimension ordering convention, "Width-Height-Depth".
+
+The Xception model is only available for TensorFlow, due to its reliance on `SeparableConvolution` layers.
+
+### Model for music audio file auto-tagging (taking as input Mel-spectrograms):
+
+- [MusicTaggerCRNN](#musictaggercrnn)
 
 -----
 
-## Examples
+## Usage examples for image classification models
 
 ### Classify ImageNet classes with ResNet50
 
@@ -26,6 +33,7 @@ All of these architectures are compatible with both TensorFlow and Theano, and u
 from keras.applications.resnet50 import ResNet50
 from keras.preprocessing import image
 from keras.applications.resnet50 import preprocess_input, decode_predictions
+import numpy as np
 
 model = ResNet50(weights='imagenet')
 
@@ -36,8 +44,10 @@ x = np.expand_dims(x, axis=0)
 x = preprocess_input(x)
 
 preds = model.predict(x)
-print('Predicted:', decode_predictions(preds))
-# print: [[u'n02504458', u'African_elephant']]
+# decode the results into a list of tuples (class, description, probability)
+# (one such list for each sample in the batch)
+print('Predicted:', decode_predictions(preds, top=3)[0])
+# Predicted: [(u'n02504013', u'Indian_elephant', 0.82658225), (u'n01871265', u'tusker', 0.1122357), (u'n02504458', u'African_elephant', 0.061040461)]
 ```
 
 ### Extract features with VGG16
@@ -46,6 +56,7 @@ print('Predicted:', decode_predictions(preds))
 from keras.applications.vgg16 import VGG16
 from keras.preprocessing import image
 from keras.applications.vgg16 import preprocess_input
+import numpy as np
 
 model = VGG16(weights='imagenet', include_top=False)
 
@@ -65,6 +76,7 @@ from keras.applications.vgg19 import VGG19
 from keras.preprocessing import image
 from keras.applications.vgg19 import preprocess_input
 from keras.models import Model
+import numpy as np
 
 base_model = VGG19(weights='imagenet')
 model = Model(input=base_model.input, output=base_model.get_layer('block4_pool').output)
@@ -153,12 +165,71 @@ model = InceptionV3(input_tensor=input_tensor, weights='imagenet', include_top=T
 
 -----
 
+# Documentation for individual models
+
+- [Xception](#xception)
+- [VGG16](#vgg16)
+- [VGG19](#vgg19)
+- [ResNet50](#resnet50)
+- [InceptionV3](#inceptionv3)
+- [MusicTaggerCRNN](#musictaggercrnn)
+
+-----
+
+
+## Xception
+
+
+```python
+keras.applications.xception.Xception(include_top=True, weights='imagenet', input_tensor=None)
+```
+
+Xception V1 model, with weights pre-trained on ImageNet.
+
+On ImageNet, this model gets to a top-1 validation accuracy of 0.790
+and a top-5 validation accuracy of 0.945.
+
+Note that this model is only available for the TensorFlow backend,
+due to its reliance on `SeparableConvolution` layers. Additionally it only supports
+the dimension ordering "tf" (width, height, channels).
+
+The default input size for this model is 299x299.
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Xception: Deep Learning with Depthwise Separable Convolutions](https://arxiv.org/abs/1610.02357)
+
+### License
+
+These weights are trained by ourselves and are released under the MIT license.
+
+
+-----
+
+
 ## VGG16
 
 ```python
 keras.applications.vgg16.VGG16(include_top=True, weights='imagenet', input_tensor=None)
 ```
 
+VGG16 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
 ### Arguments
 
 - include_top: whether to include the 3 fully-connected layers at the top of the network.
@@ -186,6 +257,14 @@ These weights are ported from the ones [released by VGG at Oxford](http://www.ro
 keras.applications.vgg19.VGG19(include_top=True, weights='imagenet', input_tensor=None)
 ```
 
+
+VGG19 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
 ### Arguments
 
 - include_top: whether to include the 3 fully-connected layers at the top of the network.
@@ -214,9 +293,18 @@ These weights are ported from the ones [released by VGG at Oxford](http://www.ro
 keras.applications.resnet50.ResNet50(include_top=True, weights='imagenet', input_tensor=None)
 ```
 
+
+ResNet50 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
+
 ### Arguments
 
-- include_top: whether to include the 3 fully-connected layers at the top of the network.
+- include_top: whether to include the fully-connected layer at the top of the network.
 - weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
 - input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
 
@@ -241,9 +329,17 @@ These weights are ported from the ones [released by Kaiming He](https://github.c
 keras.applications.inception_v3.InceptionV3(include_top=True, weights='imagenet', input_tensor=None)
 ```
 
+Inception V3 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 299x299.
+
+
 ### Arguments
 
-- include_top: whether to include the 3 fully-connected layers at the top of the network.
+- include_top: whether to include the fully-connected layer at the top of the network.
 - weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
 - input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
 
@@ -258,3 +354,64 @@ A Keras model instance.
 ### License
 
 These weights are trained by ourselves and are released under the MIT license.
+
+-----
+
+## MusicTaggerCRNN
+
+
+```python
+keras.applications.music_tagger_crnn.MusicTaggerCRNN(weights='msd', input_tensor=None, include_top=True)
+```
+
+A convolutional-recurrent model taking as input a vectorized representation of the MelSpectrogram of a music track and capable of outputting the musical genre of the track. You can use `keras.applications.music_tagger_crnn.preprocess_input` to convert a sound file to a vectorized spectrogram. This requires to have installed the [Librosa](http://librosa.github.io/librosa/) library. See [the usage example](#music-tagging-and-feature-extraction-with-musictaggercrnn).
+
+### Arguments
+
+- weights: one of `None` (random initialization) or "msd" (pre-training on [Million Song Dataset](http://labrosa.ee.columbia.edu/millionsong/)).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- include_top: whether to include the 1 fully-connected layer (output layer) at the top of the network. If False, the network outputs 32-dim features.
+
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Convolutional Recurrent Neural Networks for Music Classification](https://arxiv.org/abs/1609.04243)
+
+### License
+
+These weights are ported from the ones [released by Keunwoo Choi](https://github.com/keunwoochoi/music-auto_tagging-keras) under the [MIT license](https://github.com/keunwoochoi/music-auto_tagging-keras/blob/master/LICENSE.md).
+
+### Examples: music tagging and audio feature extraction
+
+```python
+from keras.applications.music_tagger_crnn import MusicTaggerCRNN
+from keras.applications.music_tagger_crnn import preprocess_input, decode_predictions
+import numpy as np
+
+# 1. Tagging
+model = MusicTaggerCRNN(weights='msd')
+
+audio_path = 'audio_file.mp3'
+melgram = preprocess_input(audio_path)
+melgrams = np.expand_dims(melgram, axis=0)
+
+preds = model.predict(melgrams)
+print('Predicted:')
+print(decode_predictions(preds))
+# print: ('Predicted:', [[('rock', 0.097071797), ('pop', 0.042456303), ('alternative', 0.032439161), ('indie', 0.024491295), ('female vocalists', 0.016455274)]])
+
+#. 2. Feature extraction
+model = MusicTaggerCRNN(weights='msd', include_top=False)
+
+audio_path = 'audio_file.mp3'
+melgram = preprocess_input(audio_path)
+melgrams = np.expand_dims(melgram, axis=0)
+
+feats = model.predict(melgrams)
+print('Features:')
+print(feats[0, :10])
+# print: ('Features:', [-0.19160545 0.94259131 -0.9991011 0.47644514 -0.19089699 0.99033844 0.1103896 -0.00340496 0.14823607 0.59856361])
+```

docs/templates/getting-started/functional-api-guide.md

@@ -102,7 +102,7 @@ lstm_out = LSTM(32)(x)
 Here we insert the auxiliary loss, allowing the LSTM and Embedding layer to be trained smoothly even though the main loss will be much higher in the model.
 
 ```python
-auxiliary_loss = Dense(1, activation='sigmoid', name='aux_output')(lstm_out)
+auxiliary_output = Dense(1, activation='sigmoid', name='aux_output')(lstm_out)
 ```
 
 At this point, we feed into the model our auxiliary input data by concatenating it with the LSTM output:
@@ -117,13 +117,13 @@ x = Dense(64, activation='relu')(x)
 x = Dense(64, activation='relu')(x)
 
 # and finally we add the main logistic regression layer
-main_loss = Dense(1, activation='sigmoid', name='main_output')(x)
+main_output = Dense(1, activation='sigmoid', name='main_output')(x)
 ```
 
 This defines a model with two inputs and two outputs:
 
 ```python
-model = Model(input=[main_input, auxiliary_input], output=[main_loss, auxiliary_loss])
+model = Model(input=[main_input, auxiliary_input], output=[main_output, auxiliary_output])
 ```
 
 We compile the model and assign a weight of 0.2 to the auxiliary loss.

docs/templates/getting-started/sequential-model-guide.md

@@ -121,7 +121,7 @@ Before training a model, you need to configure the learning process, which is do
 
 - an optimizer. This could be the string identifier of an existing optimizer (such as `rmsprop` or `adagrad`), or an instance of the `Optimizer` class. See: [optimizers](/optimizers).
 - a loss function. This is the objective that the model will try to minimize. It can be the string identifier of an existing loss function (such as `categorical_crossentropy` or `mse`), or it can be an objective function. See: [objectives](/objectives).
-- a list of metrics. For any classification problem you will want to set this to `metrics=['accuracy']`. A metric could be the string identifier of an existing metric (only `accuracy` is supported at this point), or a custom metric function.
+- a list of metrics. For any classification problem you will want to set this to `metrics=['accuracy']`. A metric could be the string identifier of an existing metric or a custom metric function.  Custom metric function should return either a single tensor value or a dict `metric_name -> metric_value`. See: [metrics](/metrics).
 
 ```python
 # for a multi-class classification problem
@@ -137,6 +137,24 @@ model.compile(optimizer='rmsprop',
 # for a mean squared error regression problem
 model.compile(optimizer='rmsprop',
               loss='mse')
+
+# for custom metrics
+import keras.backend as K
+
+def mean_pred(y_true, y_pred):
+    return K.mean(y_pred)
+
+def false_rates(y_true, y_pred):
+    false_neg = ...
+    false_pos = ...
+    return {
+        'false_neg': false_neg,
+        'false_pos': false_pos,
+    }
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy', mean_pred, false_rates])
 ```
 
 ----

docs/templates/index.md

@@ -2,7 +2,7 @@
 
 ## You have just found Keras.
 
-Keras is a minimalist, highly modular neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
+Keras is a high-level neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
 
 Use Keras if you need a deep learning library that:
 
@@ -143,7 +143,10 @@ By default, Keras will use TensorFlow as its tensor manipulation library. [Follo
 
 ## Support
 
-You can ask questions and join the development discussion on the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+You can ask questions and join the development discussion:
+
+- On the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+- On the [Keras Gitter channel](https://gitter.im/Keras-io/Lobby).
 
 You can also post bug reports and feature requests in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
 

docs/templates/layers/writing-your-own-keras-layers.md

@@ -4,7 +4,7 @@ For simple, stateless custom operations, you are probably better off using `laye
 
 Here is the skeleton of a Keras layer. There are only three methods you need to implement:
 
-- `build(input_shape)`: this is where you will define your weights. Trainable weights should be added to the list `self.trainable_weights`. Other attributes of note are: `self.non_trainable_weights` (list) and `self.updates` (list of update tuples (tensor, new_tensor)). For an example of how to use `non_trainable_weights` and `updates`, see the code for the `BatchNormalization` layer.
+- `build(input_shape)`: this is where you will define your weights. Trainable weights should be added to the list `self.trainable_weights`. Other attributes of note are: `self.non_trainable_weights` (list) and `self.updates` (list of update tuples (tensor, new_tensor)). For an example of how to use `non_trainable_weights` and `updates`, see the code for the `BatchNormalization` layer.  This method must set `self.built = True`, which can be done by calling `super([Layer], self).build()`.
 - `call(x)`: this is where the layer's logic lives. Unless you want your layer to support masking, you only have to care about the first argument passed to `call`: the input tensor.
 - `get_output_shape_for(input_shape)`: in case your layer modifies the shape of its input, you should specify here the shape transformation logic. This allows Keras to do automatic shape inference.
 
@@ -23,6 +23,7 @@ class MyLayer(Layer):
         initial_weight_value = np.random.random((input_dim, output_dim))
         self.W = K.variable(initial_weight_value)
         self.trainable_weights = [self.W]
+        super(MyLayer, self).build()  # be sure you call this somewhere!
 
     def call(self, x, mask=None):
         return K.dot(x, self.W)
@@ -31,4 +32,4 @@ class MyLayer(Layer):
         return (input_shape[0], self.output_dim)
 ```
 
-The existing Keras layers provide ample examples of how to implement almost anything. Never hesitate to read the source code!
+The existing Keras layers provide ample examples of how to implement almost anything. Never hesitate to read the source code!

docs/templates/metrics.md

@@ -0,0 +1,51 @@
+
+## Usage of metrics
+
+A metric is a function that is used to judge the performance of your model. Metric functions are to be supplied in the `metrics` parameter when a model is compiled.
+
+A metric function is similar to an [objective function](/objectives), except that the results from evaluating a metric are not used when training the model.
+
+You can either pass the name of an existing metric, or pass a Theano/TensorFlow symbolic function (see [Custom metrics](#custom-metrics)).
+
+#### Arguments
+  - __y_true__: True labels. Theano/TensorFlow tensor.
+  -  __y_pred__: Predictions. Theano/TensorFlow tensor of the same shape as y_true.
+
+#### Returns
+  Single tensor value representing the mean of the output array across all
+  datapoints.
+
+----
+
+## Available metrics
+
+
+{{autogenerated}}
+
+----
+
+## Custom metrics
+
+Custom metrics can be defined and passed via the compilation step. The
+function would need to take `(y_true, y_pred)` as arguments and return
+either a single tensor value or a dict `metric_name -> metric_value`.
+
+```python
+# for custom metrics
+import keras.backend as K
+
+def mean_pred(y_true, y_pred):
+    return K.mean(y_pred)
+
+def false_rates(y_true, y_pred):
+    false_neg = ...
+    false_pos = ...
+    return {
+        'false_neg': false_neg,
+        'false_pos': false_pos,
+    }
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy', mean_pred, false_rates])
+```

docs/templates/objectives.md

@@ -30,3 +30,11 @@ For a few examples of such functions, check out the [objectives source](https://
 - __kullback_leibler_divergence__ / __kld__: Information gain from a predicted probability distribution Q to a true probability distribution P. Gives a measure of difference between both distributions.
 - __poisson__: Mean of `(predictions - targets * log(predictions))`
 - __cosine_proximity__: The opposite (negative) of the mean cosine proximity between predictions and targets.
+
+**Note**: when using the `categorical_crossentropy` objective, your targets should be in categorical format (e.g. if you have 10 classes, the target for each sample should be a 10-dimensional vector that is all-zeros expect for a 1 at the index corresponding to the class of the sample). In order to convert *integer targets* into *categorical targets*, you can use the Keras utility `to_categorical`:
+
+```python
+from keras.utils.np_utils import to_categorical
+
+categorical_labels = to_categorical(int_labels, nb_classes=None)
+```

docs/templates/preprocessing/image.md

@@ -47,7 +47,7 @@ Generate batches of tensor image data with real-time data augmentation. The data
         "th" mode means that the images should have shape `(samples, channels, width, height)`.
         It defaults to the `image_dim_ordering` value found in your
         Keras config file at `~/.keras/keras.json`.
-        If you never set it, then it will be "th".
+        If you never set it, then it will be "tf".
 
 - __Methods__:
     - __fit(X)__: Compute the internal data stats related to the data-dependent transformations, based on an array of sample data.
@@ -56,12 +56,14 @@ Generate batches of tensor image data with real-time data augmentation. The data
             - __X__: sample data.
             - __augment__: Boolean (default: False). Whether to fit on randomly augmented samples.
             - __rounds__: int (default: 1). If augment, how many augmentation passes over the data to use.
+            - __seed__: int (default: None). Random seed.
     - __flow(X, y)__: Takes numpy data & label arrays, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
         - __Arguments__:
             - __X__: data.
             - __y__: labels.
             - __batch_size__: int (default: 32).
             - __shuffle__: boolean (defaut: True).
+            - __seed__: int (default: None).
             - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
             - __save_prefix__: str (default: `''`). Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
             - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "jpeg".
@@ -77,7 +79,7 @@ Generate batches of tensor image data with real-time data augmentation. The data
             - __class_mode__: one of "categorical", "binary", "sparse" or None. Default: "categorical". Determines the type of label arrays that are returned: "categorical" will be 2D one-hot encoded labels, "binary" will be 1D binary labels, "sparse" will be 1D integer labels. If None, no labels are returned (the generator will only yield batches of image data, which is useful to use `model.predict_generator()`, `model.evaluate_generator()`, etc.).
             - __batch_size__: size of the batches of data (default: 32).
             - __shuffle__: whether to shuffle the data (default: True)
-            - __seed__: optional random seed for shuffling.
+            - __seed__: optional random seed for shuffling and transformations.
             - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
             - __save_prefix__: str. Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
             - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "jpeg".
@@ -151,3 +153,40 @@ model.fit_generator(
         validation_data=validation_generator,
         nb_val_samples=800)
 ```
+
+Example of transforming images and masks together.
+
+```python
+# we create two instances with the same arguments
+data_gen_args = dict(featurewise_center=True,
+                     featurewise_std_normalization=True,
+                     rotation_range=90.,
+                     width_shift_range=0.1,
+                     height_shift_range=0.1,
+                     zoom_range=0.2)
+image_datagen = ImageDataGenerator(**data_gen_args)
+mask_datagen = ImageDataGenerator(**data_gen_args)
+
+# Provide the same seed and keyword arguments to the fit and flow methods
+seed = 1
+image_datagen.fit(images, augment=True, seed=seed)
+mask_datagen.fit(masks, augment=True, seed=seed)
+
+image_generator = image_datagen.flow_from_directory(
+    'data/images',
+    class_mode=None,
+    seed=seed)
+
+mask_generator = mask_datagen.flow_from_directory(
+    'data/masks',
+    class_mode=None,
+    seed=seed)
+
+# combine generators into one which yields image and masks
+train_generator = zip(image_generator, mask_generator)
+
+model.fit_generator(
+    train_generator,
+    samples_per_epoch=2000,
+    nb_epoch=50)
+```

examples/README.md

@@ -0,0 +1,97 @@
+# Keras examples directory
+
+[addition_rnn.py](addition_rnn.py)
+Implementation of sequence to sequence learning for performing addition of two numbers (as strings).
+
+[antirectifier.py](antirectifier.py)
+Demonstrates how to write custom layers for Keras.
+
+[babi_memnn.py](babi_memnn.py)
+Trains a memory network on the bAbI dataset for reading comprehension.
+
+[babi_rnn.py](babi_rnn.py)
+Trains a two-branch recurrent network on the bAbI dataset for reading comprehension.
+
+[cifar10_cnn.py](cifar10_cnn.py)
+Trains a simple deep CNN on the CIFAR10 small images dataset.
+
+[conv_filter_visualization.py](conv_filter_visualization.py)
+Visualization of the filters of VGG16, via gradient ascent in input space.
+
+[conv_lstm.py](conv_lstm.py)
+Demonstrates the use of a convolutional LSTM network.
+
+[deep_dream.py](deep_dream.py)
+Deep Dreams in Keras.
+
+[image_ocr.py](image_ocr.py)
+Trains a convolutional stack followed by a recurrent stack and a CTC logloss function to perform optical character recognition (OCR).
+
+[imdb_bidirectional_lstm.py](imdb_bidirectional_lstm.py)
+Trains a Bidirectional LSTM on the IMDB sentiment classification task.
+
+[imdb_cnn.py](imdb_cnn.py)
+Demonstrates the use of Convolution1D for text classification.
+
+[imdb_cnn_lstm.py](imdb_cnn_lstm.py)
+Trains a convolutional stack followed by a recurrent stack network on the IMDB sentiment classification task.
+
+[imdb_fasttext.py](imdb_fasttext.py)
+Trains a FastText model on the IMDB sentiment classification task.
+
+[imdb_lstm.py](imdb_lstm.py)
+Trains a LSTM on the IMDB sentiment classification task.
+
+[lstm_benchmark.py](lstm_benchmark.py)
+Compares different LSTM implementations on the IMDB sentiment classification task.
+
+[lstm_text_generation.py](lstm_text_generation.py)
+Generates text from Nietzsche's writings.
+
+[mnist_cnn.py](mnist_cnn.py)
+Trains a simple convnet on the MNIST dataset.
+
+[mnist_hierarchical_rnn.py](mnist_hierarchical_rnn.py)
+Trains a Hierarchical RNN (HRNN) to classify MNIST digits.
+
+[mnist_irnn.py](mnist_irnn.py)
+Reproduction of the IRNN experiment with pixel-by-pixel sequential MNIST in "A Simple Way to Initialize Recurrent Networks of Rectified Linear Units" by Le et al.
+
+[mnist_mlp.py](mnist_mlp.py)
+Trains a simple deep multi-layer perceptron on the MNIST dataset.
+
+[mnist_net2net.py](mnist_net2net.py)
+Reproduction of the Net2Net experiment with MNIST in "Net2Net: Accelerating Learning via Knowledge Transfer".
+
+[mnist_siamese_graph.py](mnist_siamese_graph.py)
+Trains a Siamese multi-layer perceptron on pairs of digits from the MNIST dataset.
+
+[mnist_sklearn_wrapper.py](mnist_sklearn_wrapper.py)
+Demonstrates how to use the sklearn wrapper.
+
+[mnist_swwae.py](mnist_swwae.py)
+Trains a Stacked What-Where AutoEncoder built on residual blocks on the MNIST dataset.
+
+[mnist_transfer_cnn.py](mnist_transfer_cnn.py)
+Transfer learning toy example.
+
+[neural_doodle.py](neural_doodle.py)
+Neural doodle.
+
+[neural_style_transfer.py](neural_style_transfer.py)
+Neural style transfer.
+
+[pretrained_word_embeddings.py](pretrained_word_embeddings.py)
+Loads pre-trained word embeddings (GloVe embeddings) into a frozen Keras Embedding layer, and uses it to train a text classification model on the 20 Newsgroup dataset.
+
+[reuters_mlp.py](reuters_mlp.py)
+Trains and evaluate a simple MLP on the Reuters newswire topic classification task.
+
+[stateful_lstm.py](stateful_lstm.py)
+Demonstrates how to use stateful RNNs to model long sequences efficiently.
+
+[variational_autoencoder.py](variational_autoencoder.py)
+Demonstrates how to build a variational autoencoder.
+
+[variational_autoencoder_deconv.py](variational_autoencoder_deconv.py)
+Demonstrates how to build a variational autoencoder with Keras using deconvolution layers.

examples/conv_lstm.py

@@ -0,0 +1,142 @@
+""" This script demonstrates the use of a convolutional LSTM network.
+This network is used to predict the next frame of an artificially
+generated movie which contains moving squares.
+"""
+from keras.models import Sequential
+from keras.layers.convolutional import Convolution3D
+from keras.layers.convolutional_recurrent import ConvLSTM2D
+from keras.layers.normalization import BatchNormalization
+import numpy as np
+import pylab as plt
+
+# We create a layer which take as input movies of shape
+# (n_frames, width, height, channels) and returns a movie
+# of identical shape.
+
+seq = Sequential()
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   input_shape=(None, 40, 40, 1),
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(Convolution3D(nb_filter=1, kernel_dim1=1, kernel_dim2=3,
+                      kernel_dim3=3, activation='sigmoid',
+                      border_mode='same', dim_ordering='tf'))
+
+seq.compile(loss='binary_crossentropy', optimizer='adadelta')
+
+
+# Artificial data generation:
+# Generate movies with 3 to 7 moving squares inside.
+# The squares are of shape 1x1 or 2x2 pixels,
+# which move linearly over time.
+# For convenience we first create movies with bigger width and height (80x80)
+# and at the end we select a 40x40 window.
+
+def generate_movies(n_samples=1200, n_frames=15):
+    row = 80
+    col = 80
+    noisy_movies = np.zeros((n_samples, n_frames, row, col, 1), dtype=np.float)
+    shifted_movies = np.zeros((n_samples, n_frames, row, col, 1),
+                              dtype=np.float)
+
+    for i in range(n_samples):
+        # Add 3 to 7 moving squares
+        n = np.random.randint(3, 8)
+
+        for j in range(n):
+            # Initial position
+            xstart = np.random.randint(20, 60)
+            ystart = np.random.randint(20, 60)
+            # Direction of motion
+            directionx = np.random.randint(0, 3) - 1
+            directiony = np.random.randint(0, 3) - 1
+
+            # Size of the square
+            w = np.random.randint(2, 4)
+
+            for t in range(n_frames):
+                x_shift = xstart + directionx * t
+                y_shift = ystart + directiony * t
+                noisy_movies[i, t, x_shift - w: x_shift + w,
+                             y_shift - w: y_shift + w, 0] += 1
+
+                # Make it more robust by adding noise.
+                # The idea is that if during inference,
+                # the value of the pixel is not exactly one,
+                # we need to train the network to be robust and still
+                # consider it as a pixel belonging to a square.
+                if np.random.randint(0, 2):
+                    noise_f = (-1)**np.random.randint(0, 2)
+                    noisy_movies[i, t,
+                                 x_shift - w - 1: x_shift + w + 1,
+                                 y_shift - w - 1: y_shift + w + 1,
+                                 0] += noise_f * 0.1
+
+                # Shift the ground truth by 1
+                x_shift = xstart + directionx * (t + 1)
+                y_shift = ystart + directiony * (t + 1)
+                shifted_movies[i, t, x_shift - w: x_shift + w,
+                               y_shift - w: y_shift + w, 0] += 1
+
+    # Cut to a 40x40 window
+    noisy_movies = noisy_movies[::, ::, 20:60, 20:60, ::]
+    shifted_movies = shifted_movies[::, ::, 20:60, 20:60, ::]
+    noisy_movies[noisy_movies >= 1] = 1
+    shifted_movies[shifted_movies >= 1] = 1
+    return noisy_movies, shifted_movies
+
+# Train the network
+noisy_movies, shifted_movies = generate_movies(n_samples=1200)
+seq.fit(noisy_movies[:1000], shifted_movies[:1000], batch_size=10,
+        nb_epoch=300, validation_split=0.05)
+
+# Testing the network on one movie
+# feed it with the first 7 positions and then
+# predict the new positions
+which = 1004
+track = noisy_movies[which][:7, ::, ::, ::]
+
+for j in range(16):
+    new_pos = seq.predict(track[np.newaxis, ::, ::, ::, ::])
+    new = new_pos[::, -1, ::, ::, ::]
+    track = np.concatenate((track, new), axis=0)
+
+
+# And then compare the predictions
+# to the ground truth
+track2 = noisy_movies[which][::, ::, ::, ::]
+for i in range(15):
+    fig = plt.figure(figsize=(10, 5))
+
+    ax = fig.add_subplot(121)
+
+    if i >= 7:
+        ax.text(1, 3, 'Predictions !', fontsize=20, color='w')
+    else:
+        ax.text(1, 3, 'Inital trajectory', fontsize=20)
+
+    toplot = track[i, ::, ::, 0]
+
+    plt.imshow(toplot)
+    ax = fig.add_subplot(122)
+    plt.text(1, 3, 'Ground truth', fontsize=20)
+
+    toplot = track2[i, ::, ::, 0]
+    if i >= 2:
+        toplot = shifted_movies[which][i - 1, ::, ::, 0]
+
+    plt.imshow(toplot)
+    plt.savefig('%i_animate.png' % (i + 1))

examples/deep_dream.py

@@ -15,17 +15,16 @@ If running on CPU, prefer the TensorFlow backend (much faster).
 Example results: http://i.imgur.com/FX6ROg9.jpg
 '''
 from __future__ import print_function
-from scipy.misc import imread, imresize, imsave
+from keras.preprocessing.image import load_img, img_to_array
 import numpy as np
+from scipy.misc import imsave
 from scipy.optimize import fmin_l_bfgs_b
 import time
 import argparse
-import h5py
-import os
 
-from keras.models import Sequential
-from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
+from keras.applications import vgg16
 from keras import backend as K
+from keras.layers import Input
 
 parser = argparse.ArgumentParser(description='Deep Dreams with Keras.')
 parser.add_argument('base_image_path', metavar='base', type=str,
@@ -46,14 +45,14 @@ weights_path = 'vgg16_weights.h5'
 
 # some settings we found interesting
 saved_settings = {
-    'bad_trip': {'features': {'conv4_1': 0.05,
-                              'conv4_2': 0.01,
-                              'conv4_3': 0.01},
+    'bad_trip': {'features': {'block4_conv1': 0.05,
+                              'block4_conv2': 0.01,
+                              'block4_conv3': 0.01},
                  'continuity': 0.1,
                  'dream_l2': 0.8,
                  'jitter': 5},
-    'dreamy': {'features': {'conv5_1': 0.05,
-                            'conv5_2': 0.02},
+    'dreamy': {'features': {'block5_conv1': 0.05,
+                            'block5_conv2': 0.02},
                'continuity': 0.1,
                'dream_l2': 0.02,
                'jitter': 0},
@@ -63,73 +62,39 @@ settings = saved_settings['dreamy']
 
 # util function to open, resize and format pictures into appropriate tensors
 def preprocess_image(image_path):
-    img = imresize(imread(image_path), (img_width, img_height))
-    img = img.transpose((2, 0, 1)).astype('float64')
+    img = load_img(image_path, target_size=(img_width, img_height))
+    img = img_to_array(img)
     img = np.expand_dims(img, axis=0)
+    img = vgg16.preprocess_input(img)
     return img
 
 # util function to convert a tensor into a valid image
 def deprocess_image(x):
-    x = x.transpose((1, 2, 0))
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((3, img_width, img_height))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((img_width, img_height, 3))
+    # Remove zero-center by mean pixel
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
-# build the VGG16 network
-model = Sequential()
-model.add(ZeroPadding2D((1, 1), batch_input_shape=(1, 3, img_width, img_height)))
-first_layer = model.layers[-1]
-# this is a placeholder tensor that will contain our generated images
-dream = first_layer.input
+if K.image_dim_ordering() == 'th':
+    img_size = (3, img_width, img_height)
+else:
+    img_size = (img_width, img_height, 3)
+# this will contain our generated image
+dream = Input(batch_shape=(1,) + img_size)
 
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-# load the weights of the VGG16 networks
-# (trained on ImageNet, won the ILSVRC competition in 2014)
-# note: when there is a complete match between your model definition
-# and your weight savefile, you can simply call model.load_weights(filename)
-assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
-f = h5py.File(weights_path)
-for k in range(f.attrs['nb_layers']):
-    if k >= len(model.layers):
-        # we don't look at the last (fully-connected) layers in the savefile
-        break
-    g = f['layer_{}'.format(k)]
-    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
-    model.layers[k].set_weights(weights)
-f.close()
+# build the VGG16 network with our placeholder
+# the model will be loaded with pre-trained ImageNet weights
+model = vgg16.VGG16(input_tensor=dream,
+                    weights='imagenet', include_top=False)
 print('Model loaded.')
 
 # get the symbolic outputs of each "key" layer (we gave them unique names).
@@ -138,8 +103,16 @@ layer_dict = dict([(layer.name, layer) for layer in model.layers])
 # continuity loss util function
 def continuity_loss(x):
     assert K.ndim(x) == 4
-    a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1])
-    b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:])
+    if K.image_dim_ordering() == 'th':
+        a = K.square(x[:, :, :img_width - 1, :img_height - 1] -
+                     x[:, :, 1:, :img_height - 1])
+        b = K.square(x[:, :, :img_width - 1, :img_height - 1] -
+                     x[:, :, :img_width - 1, 1:])
+    else:
+        a = K.square(x[:, :img_width - 1, :img_height-1, :] -
+                     x[:, 1:, :img_height - 1, :])
+        b = K.square(x[:, :img_width - 1, :img_height-1, :] -
+                     x[:, :img_width - 1, 1:, :])
     return K.sum(K.pow(a + b, 1.25))
 
 # define the loss
@@ -151,12 +124,15 @@ for layer_name in settings['features']:
     x = layer_dict[layer_name].output
     shape = layer_dict[layer_name].output_shape
     # we avoid border artifacts by only involving non-border pixels in the loss
-    loss -= coeff * K.sum(K.square(x[:, :, 2: shape[2]-2, 2: shape[3]-2])) / np.prod(shape[1:])
+    if K.image_dim_ordering() == 'th':
+        loss -= coeff * K.sum(K.square(x[:, :, 2: shape[2] - 2, 2: shape[3] - 2])) / np.prod(shape[1:])
+    else:
+        loss -= coeff * K.sum(K.square(x[:, 2: shape[1] - 2, 2: shape[2] - 2, :])) / np.prod(shape[1:])
 
 # add continuity loss (gives image local coherence, can result in an artful blur)
-loss += settings['continuity'] * continuity_loss(dream) / (3 * img_width * img_height)
+loss += settings['continuity'] * continuity_loss(dream) / np.prod(img_size)
 # add image L2 norm to loss (prevents pixels from taking very high values, makes image darker)
-loss += settings['dream_l2'] * K.sum(K.square(dream)) / (3 * img_width * img_height)
+loss += settings['dream_l2'] * K.sum(K.square(dream)) / np.prod(img_size)
 
 # feel free to further modify the loss as you see fit, to achieve new effects...
 
@@ -171,7 +147,7 @@ else:
 
 f_outputs = K.function([dream], outputs)
 def eval_loss_and_grads(x):
-    x = x.reshape((1, 3, img_width, img_height))
+    x = x.reshape((1,) + img_size)
     outs = f_outputs([x])
     loss_value = outs[0]
     if len(outs[1:]) == 1:
@@ -215,7 +191,7 @@ for i in range(5):
     start_time = time.time()
 
     # add a random jitter to the initial image. This will be reverted at decoding time
-    random_jitter = (settings['jitter'] * 2) * (np.random.random((3, img_width, img_height)) - 0.5)
+    random_jitter = (settings['jitter'] * 2) * (np.random.random(img_size) - 0.5)
     x += random_jitter
 
     # run L-BFGS for 7 steps
@@ -223,9 +199,9 @@ for i in range(5):
                                      fprime=evaluator.grads, maxfun=7)
     print('Current loss value:', min_val)
     # decode the dream and save it
-    x = x.reshape((3, img_width, img_height))
+    x = x.reshape(img_size)
     x -= random_jitter
-    img = deprocess_image(x)
+    img = deprocess_image(np.copy(x))
     fname = result_prefix + '_at_iteration_%d.png' % i
     imsave(fname, img)
     end_time = time.time()

examples/image_ocr.py

@@ -109,7 +109,7 @@ def paint_text(text, w, h):
     a = np.frombuffer(buf, np.uint8)
     a.shape = (h, w, 4)
     a = a[:, :, 0]  # grab single channel
-    a /= 255
+    a = a.astype(np.float32) / 255
     a = np.expand_dims(a, 0)
     a = speckle(a)
     a = image.random_rotation(a, 3 * (w - top_left_x) / w + 1)
@@ -396,7 +396,7 @@ pool_size_1 = 4
 pool_size_2 = 2
 time_dense_size = 32
 rnn_size = 512
-time_steps = img_w / (pool_size_1 * pool_size_2)
+time_steps = img_w // (pool_size_1 * pool_size_2)
 
 if K.image_dim_ordering() == 'th':
     input_shape = (1, img_h, img_w)
@@ -411,7 +411,7 @@ img_gen = TextImageGenerator(monogram_file=os.path.join(fdir, 'wordlist_mono_cle
                              minibatch_size=32,
                              img_w=img_w,
                              img_h=img_h,
-                             downsample_width=img_w / (pool_size_1 * pool_size_2) - 2,
+                             downsample_width=img_w // (pool_size_1 * pool_size_2) - 2,
                              val_split=words_per_epoch - val_words)
 
 act = 'relu'
@@ -423,7 +423,7 @@ inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='s
                       activation=act, name='conv2')(inner)
 inner = MaxPooling2D(pool_size=(pool_size_2, pool_size_2), name='max2')(inner)
 
-conv_to_rnn_dims = ((img_h / (pool_size_1 * pool_size_2)) * conv_num_filters, img_w / (pool_size_1 * pool_size_2))
+conv_to_rnn_dims = ((img_h // (pool_size_1 * pool_size_2)) * conv_num_filters, img_w // (pool_size_1 * pool_size_2))
 inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
 inner = Permute(dims=(2, 1), name='permute')(inner)
 

examples/imdb_cnn.py

@@ -12,9 +12,9 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
 from keras.models import Sequential
-from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Dense, Dropout, Activation
 from keras.layers import Embedding
-from keras.layers import Convolution1D, MaxPooling1D
+from keras.layers import Convolution1D, GlobalMaxPooling1D
 from keras.datasets import imdb
 from keras import backend as K
 
@@ -58,11 +58,7 @@ model.add(Convolution1D(nb_filter=nb_filter,
                         activation='relu',
                         subsample_length=1))
 # we use max pooling:
-model.add(MaxPooling1D(pool_length=model.output_shape[1]))
-
-# We flatten the output of the conv layer,
-# so that we can add a vanilla dense layer:
-model.add(Flatten())
+model.add(GlobalMaxPooling1D())
 
 # We add a vanilla hidden layer:
 model.add(Dense(hidden_dims))

examples/imdb_cnn_lstm.py

@@ -11,7 +11,7 @@ from keras.preprocessing import sequence
 from keras.models import Sequential
 from keras.layers import Dense, Dropout, Activation
 from keras.layers import Embedding
-from keras.layers import LSTM, GRU, SimpleRNN
+from keras.layers import LSTM
 from keras.layers import Convolution1D, MaxPooling1D
 from keras.datasets import imdb
 

examples/imdb_fasttext.py

@@ -6,8 +6,8 @@ Bags of Tricks for Efficient Text Classification
 https://arxiv.org/abs/1607.01759
 
 Results on IMDB datasets with uni and bi-gram embeddings:
-    Uni-gram: 0.8813 test accuracy after 5 epochs. 15s/epoch on i7 cpu.
-    Bi-gram : 0.9056 test accuracy after 5 epochs. 5s/epoch on GTX 1080 gpu.
+    Uni-gram: 0.8813 test accuracy after 5 epochs. 8s/epoch on i7 cpu.
+    Bi-gram : 0.9056 test accuracy after 5 epochs. 2s/epoch on GTX 980M gpu.
 '''
 
 from __future__ import print_function
@@ -16,9 +16,9 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
 from keras.models import Sequential
-from keras.layers import Dense, Flatten
+from keras.layers import Dense
 from keras.layers import Embedding
-from keras.layers import AveragePooling1D
+from keras.layers import GlobalAveragePooling1D
 from keras.datasets import imdb
 
 
@@ -119,12 +119,9 @@ model.add(Embedding(max_features,
                     embedding_dims,
                     input_length=maxlen))
 
-# we add a AveragePooling1D, which will average the embeddings
+# we add a GlobalAveragePooling1D, which will average the embeddings
 # of all words in the document
-model.add(AveragePooling1D(pool_length=model.output_shape[1]))
-
-# We flatten the output of the AveragePooling1D layer
-model.add(Flatten())
+model.add(GlobalAveragePooling1D())
 
 # We project onto a single unit output layer, and squash it with a sigmoid:
 model.add(Dense(1, activation='sigmoid'))

examples/mnist_acgan.py

@@ -0,0 +1,314 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Train an Auxiliary Classifier Generative Adversarial Network (ACGAN) on the
+MNIST dataset. See https://arxiv.org/abs/1610.09585 for more details.
+
+You should start to see reasonable images after ~5 epochs, and good images
+by ~15 epochs. You should use a GPU, as the convolution-heavy operations are
+very slow on the CPU. Prefer the TensorFlow backend if you plan on iterating, as
+the compilation time can be a blocker using Theano.
+
+Timings:
+
+Hardware           | Backend | Time / Epoch
+-------------------------------------------
+ CPU               | TF      | 3 hrs
+ Titan X (maxwell) | TF      | 4 min
+ Titan X (maxwell) | TH      | 7 min
+
+Consult https://github.com/lukedeo/keras-acgan for more information and
+example output
+"""
+from __future__ import print_function
+
+from collections import defaultdict
+import cPickle as pickle
+from PIL import Image
+
+from six.moves import range
+
+import keras.backend as K
+from keras.datasets import mnist
+from keras.layers import Input, Dense, Reshape, Flatten, Embedding, merge, Dropout
+from keras.layers.advanced_activations import LeakyReLU
+from keras.layers.convolutional import UpSampling2D, Convolution2D
+from keras.models import Sequential, Model
+from keras.optimizers import Adam
+from keras.utils.generic_utils import Progbar
+import numpy as np
+
+np.random.seed(1337)
+
+K.set_image_dim_ordering('th')
+
+
+def build_generator(latent_size):
+    # we will map a pair of (z, L), where z is a latent vector and L is a
+    # label drawn from P_c, to image space (..., 1, 28, 28)
+    cnn = Sequential()
+
+    cnn.add(Dense(1024, input_dim=latent_size, activation='relu'))
+    cnn.add(Dense(128 * 7 * 7, activation='relu'))
+    cnn.add(Reshape((128, 7, 7)))
+
+    # upsample to (..., 14, 14)
+    cnn.add(UpSampling2D(size=(2, 2)))
+    cnn.add(Convolution2D(256, 5, 5, border_mode='same',
+                          activation='relu', init='glorot_normal'))
+
+    # upsample to (..., 28, 28)
+    cnn.add(UpSampling2D(size=(2, 2)))
+    cnn.add(Convolution2D(128, 5, 5, border_mode='same',
+                          activation='relu', init='glorot_normal'))
+
+    # take a channel axis reduction
+    cnn.add(Convolution2D(1, 2, 2, border_mode='same',
+                          activation='tanh', init='glorot_normal'))
+
+    # this is the z space commonly refered to in GAN papers
+    latent = Input(shape=(latent_size, ))
+
+    # this will be our label
+    image_class = Input(shape=(1,), dtype='int32')
+
+    # 10 classes in MNIST
+    cls = Flatten()(Embedding(10, latent_size,
+                              init='glorot_normal')(image_class))
+
+    # hadamard product between z-space and a class conditional embedding
+    h = merge([latent, cls], mode='mul')
+
+    fake_image = cnn(h)
+
+    return Model(input=[latent, image_class], output=fake_image)
+
+
+def build_discriminator():
+    # build a relatively standard conv net, with LeakyReLUs as suggested in
+    # the reference paper
+    cnn = Sequential()
+
+    cnn.add(Convolution2D(32, 3, 3, border_mode='same', subsample=(2, 2),
+                          input_shape=(1, 28, 28)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(64, 3, 3, border_mode='same', subsample=(1, 1)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(128, 3, 3, border_mode='same', subsample=(2, 2)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(256, 3, 3, border_mode='same', subsample=(1, 1)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Flatten())
+
+    image = Input(shape=(1, 28, 28))
+
+    features = cnn(image)
+
+    # first output (name=generation) is whether or not the discriminator
+    # thinks the image that is being shown is fake, and the second output
+    # (name=auxiliary) is the class that the discriminator thinks the image
+    # belongs to.
+    fake = Dense(1, activation='sigmoid', name='generation')(features)
+    aux = Dense(10, activation='softmax', name='auxiliary')(features)
+
+    return Model(input=image, output=[fake, aux])
+
+if __name__ == '__main__':
+
+    # batch and latent size taken from the paper
+    nb_epochs = 50
+    batch_size = 100
+    latent_size = 100
+
+    # Adam parameters suggested in https://arxiv.org/abs/1511.06434
+    adam_lr = 0.0002
+    adam_beta_1 = 0.5
+
+    # build the discriminator
+    discriminator = build_discriminator()
+    discriminator.compile(
+        optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+        loss=['binary_crossentropy', 'sparse_categorical_crossentropy']
+    )
+
+    # build the generator
+    generator = build_generator(latent_size)
+    generator.compile(optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+                      loss='binary_crossentropy')
+
+    latent = Input(shape=(latent_size, ))
+    image_class = Input(shape=(1,), dtype='int32')
+
+    # get a fake image
+    fake = generator([latent, image_class])
+
+    # we only want to be able to train generation for the combined model
+    discriminator.trainable = False
+    fake, aux = discriminator(fake)
+    combined = Model(input=[latent, image_class], output=[fake, aux])
+
+    combined.compile(
+        optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+        loss=['binary_crossentropy', 'sparse_categorical_crossentropy']
+    )
+
+    discriminator.trainable = True
+
+    # get our mnist data, and force it to be of shape (..., 1, 28, 28) with
+    # range [-1, 1]
+    (X_train, y_train), (X_test, y_test) = mnist.load_data()
+    X_train = (X_train.astype(np.float32) - 127.5) / 127.5
+    X_train = np.expand_dims(X_train, axis=1)
+
+    X_test = (X_test.astype(np.float32) - 127.5) / 127.5
+    X_test = np.expand_dims(X_test, axis=1)
+
+    nb_train, nb_test = X_train.shape[0], X_test.shape[0]
+
+    train_history = defaultdict(list)
+    test_history = defaultdict(list)
+
+    for epoch in range(nb_epochs):
+        print('Epoch {} of {}'.format(epoch + 1, nb_epochs))
+
+        nb_batches = int(X_train.shape[0] / batch_size)
+        progress_bar = Progbar(target=nb_batches)
+
+        epoch_gen_loss = []
+        epoch_disc_loss = []
+
+        for index in range(nb_batches):
+            progress_bar.update(index)
+            # generate a new batch of noise
+            noise = np.random.uniform(-1, 1, (batch_size, latent_size))
+
+            # get a batch of real images
+            image_batch = X_train[index * batch_size:(index + 1) * batch_size]
+            label_batch = y_train[index * batch_size:(index + 1) * batch_size]
+
+            # sample some labels from p_c
+            sampled_labels = np.random.randint(0, 10, batch_size)
+
+            # generate a batch of fake images, using the generated labels as a
+            # conditioner. We reshape the sampled labels to be
+            # (batch_size, 1) so that we can feed them into the embedding
+            # layer as a length one sequence
+            generated_images = generator.predict(
+                [noise, sampled_labels.reshape((-1, 1))], verbose=0)
+
+            X = np.concatenate((image_batch, generated_images))
+            y = np.array([1] * batch_size + [0] * batch_size)
+            aux_y = np.concatenate((label_batch, sampled_labels), axis=0)
+
+            # see if the discriminator can figure itself out...
+            epoch_disc_loss.append(discriminator.train_on_batch(X, [y, aux_y]))
+
+            # make new noise. we generate 2 * batch size here such that we have
+            # the generator optimize over an identical number of images as the
+            # discriminator
+            noise = np.random.uniform(-1, 1, (2 * batch_size, latent_size))
+            sampled_labels = np.random.randint(0, 10, 2 * batch_size)
+
+            # we want to fix the discriminator and let the generator train to
+            # trick it
+            discriminator.trainable = False
+
+            # For the generator, we want all the {fake, not-fake} labels to say
+            # not-fake
+            trick = np.ones(2 * batch_size)
+
+            epoch_gen_loss.append(combined.train_on_batch(
+                [noise, sampled_labels.reshape((-1, 1))], [trick, sampled_labels]))
+
+            discriminator.trainable = True
+
+        print('\nTesting for epoch {}:'.format(epoch + 1))
+
+        # evaluate the testing loss here
+
+        # generate a new batch of noise
+        noise = np.random.uniform(-1, 1, (nb_test, latent_size))
+
+        # sample some labels from p_c and generate images from them
+        sampled_labels = np.random.randint(0, 10, nb_test)
+        generated_images = generator.predict(
+            [noise, sampled_labels.reshape((-1, 1))], verbose=False)
+
+        X = np.concatenate((X_test, generated_images))
+        y = np.array([1] * nb_test + [0] * nb_test)
+        aux_y = np.concatenate((y_test, sampled_labels), axis=0)
+
+        # see if the discriminator can figure itself out...
+        discriminator_test_loss = discriminator.evaluate(
+            X, [y, aux_y], verbose=False)
+
+        discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0)
+
+        # make new noise
+        noise = np.random.uniform(-1, 1, (2 * nb_test, latent_size))
+        sampled_labels = np.random.randint(0, 10, 2 * nb_test)
+
+        trick = np.ones(2 * nb_test)
+
+        generator_test_loss = combined.evaluate(
+            [noise, sampled_labels.reshape((-1, 1))],
+            [trick, sampled_labels], verbose=False)
+
+        generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0)
+
+        # generate an epoch report on performance
+        train_history['generator'].append(generator_train_loss)
+        train_history['discriminator'].append(discriminator_train_loss)
+
+        test_history['generator'].append(generator_test_loss)
+        test_history['discriminator'].append(discriminator_test_loss)
+
+        print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}'.format(
+            'component', *discriminator.metrics_names))
+        print('-' * 65)
+
+        ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}'
+        print(ROW_FMT.format('generator (train)',
+                             *train_history['generator'][-1]))
+        print(ROW_FMT.format('generator (test)',
+                             *test_history['generator'][-1]))
+        print(ROW_FMT.format('discriminator (train)',
+                             *train_history['discriminator'][-1]))
+        print(ROW_FMT.format('discriminator (test)',
+                             *test_history['discriminator'][-1]))
+
+        # save weights every epoch
+        generator.save_weights(
+            'params_generator_epoch_{0:03d}.hdf5'.format(epoch), True)
+        discriminator.save_weights(
+            'params_discriminator_epoch_{0:03d}.hdf5'.format(epoch), True)
+
+        # generate some digits to display
+        noise = np.random.uniform(-1, 1, (100, latent_size))
+
+        sampled_labels = np.array([
+            [i] * 10 for i in range(10)
+        ]).reshape(-1, 1)
+
+        # get a batch to display
+        generated_images = generator.predict(
+            [noise, sampled_labels], verbose=0)
+
+        # arrange them into a grid
+        img = (np.concatenate([r.reshape(-1, 28)
+                               for r in np.split(generated_images, 10)
+                               ], axis=-1) * 127.5 + 127.5).astype(np.uint8)
+
+        Image.fromarray(img).save(
+            'plot_epoch_{0:03d}_generated.png'.format(epoch))
+
+    pickle.dump({'train': train_history, 'test': test_history},
+                open('acgan-history.pkl', 'wb'))

examples/neural_doodle.py

@@ -108,10 +108,12 @@ def deprocess_image(x):
         x = x.transpose((1, 2, 0))
     else:
         x = x.reshape((img_nrows, img_ncols, 3))
-    x = x[:, :, ::-1]
+    # Remove zero-center by mean pixel
     x[:, :, 0] += 103.939
     x[:, :, 1] += 116.779
     x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 

examples/neural_style_transfer.py

@@ -91,10 +91,12 @@ def deprocess_image(x):
         x = x.transpose((1, 2, 0))
     else:
         x = x.reshape((img_nrows, img_ncols, 3))
-    x = x[:, :, ::-1]
+    # Remove zero-center by mean pixel
     x[:, :, 0] += 103.939
     x[:, :, 1] += 116.779
     x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 

examples/stateful_lstm.py

@@ -54,7 +54,6 @@ model.add(LSTM(50,
                return_sequences=True,
                stateful=True))
 model.add(LSTM(50,
-               batch_input_shape=(batch_size, tsteps, 1),
                return_sequences=False,
                stateful=True))
 model.add(Dense(1))

examples/variational_autoencoder.py

@@ -4,6 +4,7 @@ Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
 '''
 import numpy as np
 import matplotlib.pyplot as plt
+from scipy.stats import norm
 
 from keras.layers import Input, Dense, Lambda
 from keras.models import Model
@@ -16,6 +17,7 @@ original_dim = 784
 latent_dim = 2
 intermediate_dim = 256
 nb_epoch = 50
+epsilon_std = 1.0
 
 x = Input(batch_shape=(batch_size, original_dim))
 h = Dense(intermediate_dim, activation='relu')(x)
@@ -25,7 +27,8 @@ z_log_var = Dense(latent_dim)(h)
 
 def sampling(args):
     z_mean, z_log_var = args
-    epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0.)
+    epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0.,
+                              std=epsilon_std)
     return z_mean + K.exp(z_log_var / 2) * epsilon
 
 # note that "output_shape" isn't necessary with the TensorFlow backend
@@ -80,9 +83,10 @@ generator = Model(decoder_input, _x_decoded_mean)
 n = 15  # figure with 15x15 digits
 digit_size = 28
 figure = np.zeros((digit_size * n, digit_size * n))
-# we will sample n points within [-15, 15] standard deviations
-grid_x = np.linspace(-15, 15, n)
-grid_y = np.linspace(-15, 15, n)
+# linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
+# to produce values of the latent variables z, since the prior of the latent space is Gaussian
+grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
+grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
 
 for i, yi in enumerate(grid_x):
     for j, xi in enumerate(grid_y):
@@ -93,5 +97,5 @@ for i, yi in enumerate(grid_x):
                j * digit_size: (j + 1) * digit_size] = digit
 
 plt.figure(figsize=(10, 10))
-plt.imshow(figure)
+plt.imshow(figure, cmap='Greys_r')
 plt.show()

examples/variational_autoencoder_deconv.py

@@ -1,12 +1,14 @@
-'''This script demonstrates how to build a variational autoencoder with Keras and deconvolution layers.
+'''This script demonstrates how to build a variational autoencoder
+with Keras and deconvolution layers.
 
 Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
 '''
 import numpy as np
 import matplotlib.pyplot as plt
+from scipy.stats import norm
 
 from keras.layers import Input, Dense, Lambda, Flatten, Reshape
-from keras.layers import Convolution2D, Deconvolution2D, MaxPooling2D
+from keras.layers import Convolution2D, Deconvolution2D
 from keras.models import Model
 from keras import backend as K
 from keras import objectives
@@ -15,25 +17,36 @@ from keras.datasets import mnist
 # input image dimensions
 img_rows, img_cols, img_chns = 28, 28, 1
 # number of convolutional filters to use
-nb_filters = 32
+nb_filters = 64
 # convolution kernel size
 nb_conv = 3
 
-batch_size = 16
-original_dim = (img_chns, img_rows, img_cols)
+batch_size = 100
+if K.image_dim_ordering() == 'th':
+    original_img_size = (img_chns, img_rows, img_cols)
+else:
+    original_img_size = (img_rows, img_cols, img_chns)
 latent_dim = 2
 intermediate_dim = 128
-epsilon_std = 0.01
+epsilon_std = 1.0
 nb_epoch = 5
 
+x = Input(batch_shape=(batch_size,) + original_img_size)
+conv_1 = Convolution2D(img_chns, 2, 2, border_mode='same', activation='relu')(x)
+conv_2 = Convolution2D(nb_filters, 2, 2,
+                       border_mode='same', activation='relu',
+                       subsample=(2, 2))(conv_1)
+conv_3 = Convolution2D(nb_filters, nb_conv, nb_conv,
+                       border_mode='same', activation='relu',
+                       subsample=(1, 1))(conv_2)
+conv_4 = Convolution2D(nb_filters, nb_conv, nb_conv,
+                       border_mode='same', activation='relu',
+                       subsample=(1, 1))(conv_3)
+flat = Flatten()(conv_4)
+hidden = Dense(intermediate_dim, activation='relu')(flat)
 
-x = Input(batch_shape=(batch_size,) + original_dim)
-c = Convolution2D(nb_filters, nb_conv, nb_conv, border_mode='same', activation='relu')(x)
-f = Flatten()(c)
-h = Dense(intermediate_dim, activation='relu')(f)
-
-z_mean = Dense(latent_dim)(h)
-z_log_var = Dense(latent_dim)(h)
+z_mean = Dense(latent_dim)(hidden)
+z_log_var = Dense(latent_dim)(hidden)
 
 
 def sampling(args):
@@ -47,36 +60,68 @@ def sampling(args):
 z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
 
 # we instantiate these layers separately so as to reuse them later
-decoder_h = Dense(intermediate_dim, activation='relu')
-decoder_f = Dense(nb_filters*img_rows*img_cols, activation='relu')
-decoder_c = Reshape((nb_filters, img_rows, img_cols))
-decoder_mean = Deconvolution2D(img_chns, nb_conv, nb_conv,
-                               (batch_size, img_chns, img_rows, img_cols),
-                               border_mode='same')
+decoder_hid = Dense(intermediate_dim, activation='relu')
+decoder_upsample = Dense(nb_filters * 14 * 14, activation='relu')
 
-h_decoded = decoder_h(z)
-f_decoded = decoder_f(h_decoded)
-c_decoded = decoder_c(f_decoded)
-x_decoded_mean = decoder_mean(c_decoded)
+if K.image_dim_ordering() == 'th':
+    output_shape = (batch_size, nb_filters, 14, 14)
+else:
+    output_shape = (batch_size, 14, 14, nb_filters)
 
+decoder_reshape = Reshape(output_shape[1:])
+decoder_deconv_1 = Deconvolution2D(nb_filters, nb_conv, nb_conv,
+                                   output_shape,
+                                   border_mode='same',
+                                   subsample=(1, 1),
+                                   activation='relu')
+decoder_deconv_2 = Deconvolution2D(nb_filters, nb_conv, nb_conv,
+                                   output_shape,
+                                   border_mode='same',
+                                   subsample=(1, 1),
+                                   activation='relu')
+if K.image_dim_ordering() == 'th':
+    output_shape = (batch_size, nb_filters, 29, 29)
+else:
+    output_shape = (batch_size, 29, 29, nb_filters)
+decoder_deconv_3_upsamp = Deconvolution2D(nb_filters, 2, 2,
+                                          output_shape,
+                                          border_mode='valid',
+                                          subsample=(2, 2),
+                                          activation='relu')
+decoder_mean_squash = Convolution2D(img_chns, 2, 2,
+                                    border_mode='valid',
+                                    activation='sigmoid')
+
+hid_decoded = decoder_hid(z)
+up_decoded = decoder_upsample(hid_decoded)
+reshape_decoded = decoder_reshape(up_decoded)
+deconv_1_decoded = decoder_deconv_1(reshape_decoded)
+deconv_2_decoded = decoder_deconv_2(deconv_1_decoded)
+x_decoded_relu = decoder_deconv_3_upsamp(deconv_2_decoded)
+x_decoded_mean_squash = decoder_mean_squash(x_decoded_relu)
 
 def vae_loss(x, x_decoded_mean):
-    # NOTE: binary_crossentropy expects a batch_size by dim for x and x_decoded_mean, so we MUST flatten these!
+    # NOTE: binary_crossentropy expects a batch_size by dim
+    # for x and x_decoded_mean, so we MUST flatten these!
     x = K.flatten(x)
     x_decoded_mean = K.flatten(x_decoded_mean)
-    xent_loss = objectives.binary_crossentropy(x, x_decoded_mean)
+    xent_loss = img_rows * img_cols * objectives.binary_crossentropy(x, x_decoded_mean)
     kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
     return xent_loss + kl_loss
 
-vae = Model(x, x_decoded_mean)
+vae = Model(x, x_decoded_mean_squash)
 vae.compile(optimizer='rmsprop', loss=vae_loss)
 vae.summary()
 
 # train the VAE on MNIST digits
-(x_train, y_train), (x_test, y_test) = mnist.load_data()
+(x_train, _), (x_test, y_test) = mnist.load_data()
 
-x_train = x_train.astype('float32')[:, None, :, :] / 255.
-x_test = x_test.astype('float32')[:, None, :, :] / 255.
+x_train = x_train.astype('float32') / 255.
+x_train = x_train.reshape((x_train.shape[0],) + original_img_size)
+x_test = x_test.astype('float32') / 255.
+x_test = x_test.reshape((x_test.shape[0],) + original_img_size)
+
+print('x_train.shape:', x_train.shape)
 
 vae.fit(x_train, x_train,
         shuffle=True,
@@ -84,7 +129,6 @@ vae.fit(x_train, x_train,
         batch_size=batch_size,
         validation_data=(x_test, x_test))
 
-
 # build a model to project inputs on the latent space
 encoder = Model(x, z_mean)
 
@@ -97,28 +141,33 @@ plt.show()
 
 # build a digit generator that can sample from the learned distribution
 decoder_input = Input(shape=(latent_dim,))
-_h_decoded = decoder_h(decoder_input)
-_f_decoded = decoder_f(_h_decoded)
-_c_decoded = decoder_c(_f_decoded)
-_x_decoded_mean = decoder_mean(_c_decoded)
-generator = Model(decoder_input, _x_decoded_mean)
+_hid_decoded = decoder_hid(decoder_input)
+_up_decoded = decoder_upsample(_hid_decoded)
+_reshape_decoded = decoder_reshape(_up_decoded)
+_deconv_1_decoded = decoder_deconv_1(_reshape_decoded)
+_deconv_2_decoded = decoder_deconv_2(_deconv_1_decoded)
+_x_decoded_relu = decoder_deconv_3_upsamp(_deconv_2_decoded)
+_x_decoded_mean_squash = decoder_mean_squash(_x_decoded_relu)
+generator = Model(decoder_input, _x_decoded_mean_squash)
 
 # display a 2D manifold of the digits
 n = 15  # figure with 15x15 digits
 digit_size = 28
 figure = np.zeros((digit_size * n, digit_size * n))
-# we will sample n points within [-15, 15] standard deviations
-grid_x = np.linspace(-15, 15, n)
-grid_y = np.linspace(-15, 15, n)
+# linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
+# to produce values of the latent variables z, since the prior of the latent space is Gaussian
+grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
+grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
 
 for i, yi in enumerate(grid_x):
     for j, xi in enumerate(grid_y):
         z_sample = np.array([[xi, yi]])
-        x_decoded = generator.predict(z_sample)
+        z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2)
+        x_decoded = generator.predict(z_sample, batch_size=batch_size)
         digit = x_decoded[0].reshape(digit_size, digit_size)
         figure[i * digit_size: (i + 1) * digit_size,
                j * digit_size: (j + 1) * digit_size] = digit
 
 plt.figure(figsize=(10, 10))
-plt.imshow(figure)
+plt.imshow(figure, cmap='Greys_r')
 plt.show()

keras/__init__.py

@@ -15,4 +15,4 @@ from . import objectives
 from . import optimizers
 from . import regularizers
 
-__version__ = '1.1.0'
+__version__ = '1.1.2'

keras/activations.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def softmax(x):
@@ -11,8 +12,13 @@ def softmax(x):
         s = K.sum(e, axis=-1, keepdims=True)
         return e / s
     else:
-        raise Exception('Cannot apply softmax to a tensor that is not 2D or 3D. ' +
-                        'Here, ndim=' + str(ndim))
+        raise ValueError('Cannot apply softmax to a tensor '
+                         'that is not 2D or 3D. '
+                         'Here, ndim=' + str(ndim))
+
+
+def elu(x, alpha=1.0):
+    return K.elu(x, alpha)
 
 
 def softplus(x):
@@ -40,13 +46,9 @@ def hard_sigmoid(x):
 
 
 def linear(x):
-    '''
-    The function returns the variable that is passed in, so all types work.
-    '''
     return x
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier):
     if identifier is None:
         return linear

keras/applications/__init__.py

@@ -2,3 +2,4 @@ from .vgg16 import VGG16
 from .vgg19 import VGG19
 from .resnet50 import ResNet50
 from .inception_v3 import InceptionV3
+from .xception import Xception

keras/applications/audio_conv_utils.py

@@ -0,0 +1,86 @@
+import numpy as np
+from .. import backend as K
+
+
+TAGS = ['rock', 'pop', 'alternative', 'indie', 'electronic',
+        'female vocalists', 'dance', '00s', 'alternative rock', 'jazz',
+        'beautiful', 'metal', 'chillout', 'male vocalists',
+        'classic rock', 'soul', 'indie rock', 'Mellow', 'electronica',
+        '80s', 'folk', '90s', 'chill', 'instrumental', 'punk',
+        'oldies', 'blues', 'hard rock', 'ambient', 'acoustic',
+        'experimental', 'female vocalist', 'guitar', 'Hip-Hop',
+        '70s', 'party', 'country', 'easy listening',
+        'sexy', 'catchy', 'funk', 'electro', 'heavy metal',
+        'Progressive rock', '60s', 'rnb', 'indie pop',
+        'sad', 'House', 'happy']
+
+
+def librosa_exists():
+    try:
+        __import__('librosa')
+    except ImportError:
+        return False
+    else:
+        return True
+
+
+def preprocess_input(audio_path, dim_ordering='default'):
+    '''Reads an audio file and outputs a Mel-spectrogram.
+    '''
+    if dim_ordering == 'default':
+        dim_ordering = K.image_dim_ordering()
+    assert dim_ordering in {'tf', 'th'}
+
+    if librosa_exists():
+        import librosa
+    else:
+        raise RuntimeError('Librosa is required to process audio files.\n' +
+                           'Install it via `pip install librosa` \nor visit ' +
+                           'http://librosa.github.io/librosa/ for details.')
+
+    # mel-spectrogram parameters
+    SR = 12000
+    N_FFT = 512
+    N_MELS = 96
+    HOP_LEN = 256
+    DURA = 29.12
+
+    src, sr = librosa.load(audio_path, sr=SR)
+    n_sample = src.shape[0]
+    n_sample_wanted = int(DURA * SR)
+
+    # trim the signal at the center
+    if n_sample < n_sample_wanted:  # if too short
+        src = np.hstack((src, np.zeros((int(DURA * SR) - n_sample,))))
+    elif n_sample > n_sample_wanted:  # if too long
+        src = src[(n_sample - n_sample_wanted) / 2:
+                  (n_sample + n_sample_wanted) / 2]
+
+    logam = librosa.logamplitude
+    melgram = librosa.feature.melspectrogram
+    x = logam(melgram(y=src, sr=SR, hop_length=HOP_LEN,
+                      n_fft=N_FFT, n_mels=N_MELS) ** 2,
+              ref_power=1.0)
+
+    if dim_ordering == 'th':
+        x = np.expand_dims(x, axis=0)
+    elif dim_ordering == 'tf':
+        x = np.expand_dims(x, axis=3)
+    return x
+
+
+def decode_predictions(preds, top_n=5):
+    '''Decode the output of a music tagger model.
+
+    # Arguments
+        preds: 2-dimensional numpy array
+        top_n: integer in [0, 50], number of items to show
+
+    '''
+    assert len(preds.shape) == 2 and preds.shape[1] == 50
+    results = []
+    for pred in preds:
+        result = zip(TAGS, pred)
+        result = sorted(result, key=lambda x: x[1], reverse=True)
+        results.append(result[:top_n])
+    return results

keras/applications/imagenet_utils.py

@@ -14,30 +14,38 @@ def preprocess_input(x, dim_ordering='default'):
     assert dim_ordering in {'tf', 'th'}
 
     if dim_ordering == 'th':
+        # 'RGB'->'BGR'
+        x = x[:, ::-1, :, :]
+        # Zero-center by mean pixel
         x[:, 0, :, :] -= 103.939
         x[:, 1, :, :] -= 116.779
         x[:, 2, :, :] -= 123.68
-        # 'RGB'->'BGR'
-        x = x[:, ::-1, :, :]
     else:
+        # 'RGB'->'BGR'
+        x = x[:, :, :, ::-1]
+        # Zero-center by mean pixel
         x[:, :, :, 0] -= 103.939
         x[:, :, :, 1] -= 116.779
         x[:, :, :, 2] -= 123.68
-        # 'RGB'->'BGR'
-        x = x[:, :, :, ::-1]
     return x
 
 
-def decode_predictions(preds):
+def decode_predictions(preds, top=5):
     global CLASS_INDEX
-    assert len(preds.shape) == 2 and preds.shape[1] == 1000
+    if len(preds.shape) != 2 or preds.shape[1] != 1000:
+        raise ValueError('`decode_predictions` expects '
+                         'a batch of predictions '
+                         '(i.e. a 2D array of shape (samples, 1000)). '
+                         'Found array with shape: ' + str(preds.shape))
     if CLASS_INDEX is None:
         fpath = get_file('imagenet_class_index.json',
                          CLASS_INDEX_PATH,
                          cache_subdir='models')
         CLASS_INDEX = json.load(open(fpath))
-    indices = np.argmax(preds, axis=-1)
     results = []
-    for i in indices:
-        results.append(CLASS_INDEX[str(i)])
+    for pred in preds:
+        top_indices = pred.argsort()[-top:][::-1]
+        result = [tuple(CLASS_INDEX[str(i)]) + (pred[i],) for i in top_indices]
+        result.sort(key=lambda x: x[2], reverse=True)
+        results.append(result)
     return results

keras/applications/inception_v3.py

@@ -7,8 +7,8 @@ only gets to 7.8% (same as a fully-converged ResNet 50).
 For comparison, VGG16 only gets to 9.9%, quite a bit worse.
 
 Also, do note that the input image format for this model is different than for
-other models (299x299 instead of 224x224), and that the input preprocessing function
-is also different.
+the VGG16 and ResNet models (299x299 instead of 224x224), and that the input preprocessing function
+is also different (same as Xception).
 
 # Reference:
 
@@ -76,8 +76,8 @@ def InceptionV3(include_top=True, weights='imagenet',
     Note that the default input image size for this model is 299x299.
 
     # Arguments
-        include_top: whether to include the 3 fully-connected
-            layers at the top of the network.
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
         weights: one of `None` (random initialization)
             or "imagenet" (pre-training on ImageNet).
         input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)

keras/applications/music_tagger_crnn.py

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+'''MusicTaggerCRNN model for Keras.
+
+# Reference:
+
+- [Music-auto_tagging-keras](https://github.com/keunwoochoi/music-auto_tagging-keras)
+
+'''
+from __future__ import print_function
+from __future__ import absolute_import
+
+from .. import backend as K
+from ..layers import Input, Dense
+from ..models import Model
+from ..layers import Dense, Dropout, Reshape, Permute
+from ..layers.convolutional import Convolution2D
+from ..layers.convolutional import MaxPooling2D, ZeroPadding2D
+from ..layers.normalization import BatchNormalization
+from ..layers.advanced_activations import ELU
+from ..layers.recurrent import GRU
+from ..utils.data_utils import get_file
+from ..utils.layer_utils import convert_all_kernels_in_model
+from .audio_conv_utils import decode_predictions, preprocess_input
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.3/music_tagger_crnn_weights_tf_kernels_th_dim_ordering.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.3/music_tagger_crnn_weights_tf_kernels_tf_dim_ordering.h5'
+
+
+def MusicTaggerCRNN(weights='msd', input_tensor=None,
+                    include_top=True):
+    '''Instantiate the MusicTaggerCRNN architecture,
+    optionally loading weights pre-trained
+    on Million Song Dataset. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    For preparing mel-spectrogram input, see
+    `audio_conv_utils.py` in [applications](https://github.com/fchollet/keras/tree/master/keras/applications).
+    You will need to install [Librosa](http://librosa.github.io/librosa/)
+    to use it.
+
+    # Arguments
+        weights: one of `None` (random initialization)
+            or "msd" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+        include_top: whether to include the 1 fully-connected
+            layer (output layer) at the top of the network.
+            If False, the network outputs 32-dim features.
+
+
+    # Returns
+        A Keras model instance.
+    '''
+    if weights not in {'msd', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `msd` '
+                         '(pre-training on Million Song Dataset).')
+
+    # Determine proper input shape
+    if K.image_dim_ordering() == 'th':
+        input_shape = (1, 96, 1366)
+    else:
+        input_shape = (96, 1366, 1)
+
+    if input_tensor is None:
+        melgram_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            melgram_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            melgram_input = input_tensor
+
+    # Determine input axis
+    if K.image_dim_ordering() == 'th':
+        channel_axis = 1
+        freq_axis = 2
+        time_axis = 3
+    else:
+        channel_axis = 3
+        freq_axis = 1
+        time_axis = 2
+
+    # Input block
+    x = ZeroPadding2D(padding=(0, 37))(melgram_input)
+    x = BatchNormalization(axis=time_axis, name='bn_0_freq')(x)
+
+    # Conv block 1
+    x = Convolution2D(64, 3, 3, border_mode='same', name='conv1')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn1')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x)
+
+    # Conv block 2
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv2')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn2')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(x)
+
+    # Conv block 3
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv3')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn3')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(x)
+
+    # Conv block 4
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv4')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn4')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(x)
+
+    # reshaping
+    if K.image_dim_ordering() == 'th':
+        x = Permute((3, 1, 2))(x)
+    x = Reshape((15, 128))(x)
+
+    # GRU block 1, 2, output
+    x = GRU(32, return_sequences=True, name='gru1')(x)
+    x = GRU(32, return_sequences=False, name='gru2')(x)
+
+    if include_top:
+        x = Dense(50, activation='sigmoid', name='output')(x)
+
+    # Create model
+    model = Model(melgram_input, x)
+    if weights is None:
+        return model
+    else:
+        # Load weights
+        if K.image_dim_ordering() == 'tf':
+            weights_path = get_file('music_tagger_crnn_weights_tf_kernels_tf_dim_ordering.h5',
+                                    TF_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        else:
+            weights_path = get_file('music_tagger_crnn_weights_tf_kernels_th_dim_ordering.h5',
+                                    TH_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        model.load_weights(weights_path, by_name=True)
+        if K.backend() == 'theano':
+            convert_all_kernels_in_model(model)
+        return model

keras/applications/xception.py

@@ -0,0 +1,210 @@
+# -*- coding: utf-8 -*-
+'''Xception V1 model for Keras.
+
+On ImageNet, this model gets to a top-1 validation accuracy of 0.790
+and a top-5 validation accuracy of 0.945.
+
+Do note that the input image format for this model is different than for
+the VGG16 and ResNet models (299x299 instead of 224x224),
+and that the input preprocessing function
+is also different (same as Inception V3).
+
+Also do note that this model is only available for the TensorFlow backend,
+due to its reliance on `SeparableConvolution` layers.
+
+# Reference:
+
+- [Xception: Deep Learning with Depthwise Separable Convolutions](https://arxiv.org/abs/1610.02357)
+
+'''
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..models import Model
+from ..layers import Dense, Input, BatchNormalization, Activation, merge
+from ..layers import Conv2D, SeparableConv2D, MaxPooling2D, GlobalAveragePooling2D
+from ..utils.data_utils import get_file
+from .. import backend as K
+from .imagenet_utils import decode_predictions
+
+
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def Xception(include_top=True, weights='imagenet',
+             input_tensor=None):
+    '''Instantiate the Xception architecture,
+    optionally loading weights pre-trained
+    on ImageNet. This model is available for TensorFlow only,
+    and can only be used with inputs following the TensorFlow
+    dimension ordering `(width, height, channels)`.
+    You should set `image_dim_ordering="tf"` in your Keras config
+    located at ~/.keras/keras.json.
+
+    Note that the default input image size for this model is 299x299.
+
+    # Arguments
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+
+    # Returns
+        A Keras model instance.
+    '''
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+    if K.backend() != 'tensorflow':
+        raise Exception('The Xception model is only available with '
+                        'the TensorFlow backend.')
+    if K.image_dim_ordering() != 'tf':
+        warnings.warn('The Xception model is only available for the '
+                      'input dimension ordering "tf" '
+                      '(width, height, channels). '
+                      'However your settings specify the default '
+                      'dimension ordering "th" (channels, width, height). '
+                      'You should set `image_dim_ordering="tf"` in your Keras '
+                      'config located at ~/.keras/keras.json. '
+                      'The model being returned right now will expect inputs '
+                      'to follow the "tf" dimension ordering.')
+        K.set_image_dim_ordering('tf')
+        old_dim_ordering = 'th'
+    else:
+        old_dim_ordering = None
+
+    # Determine proper input shape
+    if include_top:
+        input_shape = (299, 299, 3)
+    else:
+        input_shape = (None, None, 3)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+
+    x = Conv2D(32, 3, 3, subsample=(2, 2), bias=False, name='block1_conv1')(img_input)
+    x = BatchNormalization(name='block1_conv1_bn')(x)
+    x = Activation('relu', name='block1_conv1_act')(x)
+    x = Conv2D(64, 3, 3, bias=False, name='block1_conv2')(x)
+    x = BatchNormalization(name='block1_conv2_bn')(x)
+    x = Activation('relu', name='block1_conv2_act')(x)
+
+    residual = Conv2D(128, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = SeparableConv2D(128, 3, 3, border_mode='same', bias=False, name='block2_sepconv1')(x)
+    x = BatchNormalization(name='block2_sepconv1_bn')(x)
+    x = Activation('relu', name='block2_sepconv2_act')(x)
+    x = SeparableConv2D(128, 3, 3, border_mode='same', bias=False, name='block2_sepconv2')(x)
+    x = BatchNormalization(name='block2_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block2_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(256, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block3_sepconv1_act')(x)
+    x = SeparableConv2D(256, 3, 3, border_mode='same', bias=False, name='block3_sepconv1')(x)
+    x = BatchNormalization(name='block3_sepconv1_bn')(x)
+    x = Activation('relu', name='block3_sepconv2_act')(x)
+    x = SeparableConv2D(256, 3, 3, border_mode='same', bias=False, name='block3_sepconv2')(x)
+    x = BatchNormalization(name='block3_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block3_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(728, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block4_sepconv1_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block4_sepconv1')(x)
+    x = BatchNormalization(name='block4_sepconv1_bn')(x)
+    x = Activation('relu', name='block4_sepconv2_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block4_sepconv2')(x)
+    x = BatchNormalization(name='block4_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block4_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    for i in range(8):
+        residual = x
+        prefix = 'block' + str(i + 5)
+
+        x = Activation('relu', name=prefix + '_sepconv1_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv1')(x)
+        x = BatchNormalization(name=prefix + '_sepconv1_bn')(x)
+        x = Activation('relu', name=prefix + '_sepconv2_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv2')(x)
+        x = BatchNormalization(name=prefix + '_sepconv2_bn')(x)
+        x = Activation('relu', name=prefix + '_sepconv3_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv3')(x)
+        x = BatchNormalization(name=prefix + '_sepconv3_bn')(x)
+
+        x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(1024, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block13_sepconv1_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block13_sepconv1')(x)
+    x = BatchNormalization(name='block13_sepconv1_bn')(x)
+    x = Activation('relu', name='block13_sepconv2_act')(x)
+    x = SeparableConv2D(1024, 3, 3, border_mode='same', bias=False, name='block13_sepconv2')(x)
+    x = BatchNormalization(name='block13_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block13_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    x = SeparableConv2D(1536, 3, 3, border_mode='same', bias=False, name='block14_sepconv1')(x)
+    x = BatchNormalization(name='block14_sepconv1_bn')(x)
+    x = Activation('relu', name='block14_sepconv1_act')(x)
+
+    x = SeparableConv2D(2048, 3, 3, border_mode='same', bias=False, name='block14_sepconv2')(x)
+    x = BatchNormalization(name='block14_sepconv2_bn')(x)
+    x = Activation('relu', name='block14_sepconv2_act')(x)
+
+    if include_top:
+        x = GlobalAveragePooling2D(name='avg_pool')(x)
+        x = Dense(1000, activation='softmax', name='predictions')(x)
+
+    # Create model
+    model = Model(img_input, x)
+
+    # load weights
+    if weights == 'imagenet':
+        if include_top:
+            weights_path = get_file('xception_weights_tf_dim_ordering_tf_kernels.h5',
+                                    TF_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        else:
+            weights_path = get_file('xception_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                    TF_WEIGHTS_PATH_NO_TOP,
+                                    cache_subdir='models')
+        model.load_weights(weights_path)
+
+    if old_dim_ordering:
+        K.set_image_dim_ordering(old_dim_ordering)
+    return model
+
+
+def preprocess_input(x):
+    x /= 255.
+    x -= 0.5
+    x *= 2.
+    return x

keras/backend/__init__.py

@@ -23,7 +23,12 @@ _keras_dir = os.path.join(_keras_base_dir, '.keras')
 if not os.path.exists(_keras_dir):
     os.makedirs(_keras_dir)
 
-_BACKEND = 'tensorflow'
+# Set theano as default backend for Windows users since tensorflow is not available for Windows yet.
+if os.name == 'nt':
+    _BACKEND = 'theano'
+else:
+    _BACKEND = 'tensorflow'
+
 _config_path = os.path.expanduser(os.path.join(_keras_dir, 'keras.json'))
 if os.path.exists(_config_path):
     _config = json.load(open(_config_path))

keras/backend/tensorflow_backend.py

@@ -1,34 +1,52 @@
 import tensorflow as tf
+
 from tensorflow.python.training import moving_averages
+from tensorflow.python.ops import tensor_array_ops
+from tensorflow.python.ops import control_flow_ops
 try:
-    import tensorflow.contrib.ctc as ctc
-except ImportError:
     from tensorflow.python.ops import ctc_ops as ctc
+except ImportError:
+    import tensorflow.contrib.ctc as ctc
+
 import numpy as np
 import os
 import copy
 import warnings
-from .common import _FLOATX, _EPSILON, _IMAGE_DIM_ORDERING, reset_uids
+from .common import _FLOATX, _EPSILON, image_dim_ordering, reset_uids
 py_all = all
 
 # INTERNAL UTILS
 
+# This is the default internal TF session used by Keras.
+# It can be set manually via `set_session(sess)`.
 _SESSION = None
-_LEARNING_PHASE = tf.placeholder(dtype='uint8', name='keras_learning_phase')  # 0 = test, 1 = train
+# This dictionary holds a mapping {graph: learning_phase}.
+# A learning phase is a bool tensor used to run Keras models in
+# either train mode (learning_phase == 1) or test mode (learning_phase == 0).
+_GRAPH_LEARNING_PHASES = {}
+# This boolean flag can be set to True to leave variable initialization
+# up to the user.
+# Change its value via `manual_variable_initialization(value)`.
 _MANUAL_VAR_INIT = False
 
 
 def clear_session():
+    '''Destroys the current TF graph and creates a new one.
+
+    Useful to avoid clutter from old models / layers.
+    '''
     global _SESSION
-    global _LEARNING_PHASE
+    global _GRAPH_LEARNING_PHASES
     tf.reset_default_graph()
     reset_uids()
     _SESSION = None
-    _LEARNING_PHASE = tf.placeholder(dtype='uint8', name='keras_learning_phase')
+    phase = tf.placeholder(dtype='bool', name='keras_learning_phase')
+    _GRAPH_LEARNING_PHASES[tf.get_default_graph()] = phase
 
 
 def manual_variable_initialization(value):
-    '''Whether variables should be initialized
+    '''Returns a boolean:
+    whether variables should be initialized
     as they are instantiated (default), or if
     the user should handle the initialization
     (e.g. via tf.initialize_all_variables()).
@@ -40,19 +58,27 @@ def manual_variable_initialization(value):
 def learning_phase():
     '''Returns the learning phase flag.
 
-    The learning phase flag is an integer tensor (0 = test, 1 = train)
+    The learning phase flag is a bool tensor (0 = test, 1 = train)
     to be passed as input to any Keras function
     that uses a different behavior at train time and test time.
     '''
-    return _LEARNING_PHASE
+    graph = tf.get_default_graph()
+    if graph not in _GRAPH_LEARNING_PHASES:
+        phase = tf.placeholder(dtype='bool',
+                               name='keras_learning_phase')
+        _GRAPH_LEARNING_PHASES[graph] = phase
+    return _GRAPH_LEARNING_PHASES[graph]
 
 
 def set_learning_phase(value):
-    global _LEARNING_PHASE
+    '''Sets the learning phase to a fixed value,
+    either 0 or 1 (integers).
+    '''
+    global _GRAPH_LEARNING_PHASES
     if value not in {0, 1}:
         raise ValueError('Expected learning phase to be '
                          '0 or 1.')
-    _LEARNING_PHASE = value
+    _GRAPH_LEARNING_PHASES[tf.get_default_graph()] = value
 
 
 def get_session():
@@ -70,15 +96,20 @@ def get_session():
     '''
     global _SESSION
     if tf.get_default_session() is not None:
-        return tf.get_default_session()
-    if _SESSION is None:
-        if not os.environ.get('OMP_NUM_THREADS'):
-            _SESSION = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
-        else:
-            nb_thread = int(os.environ.get('OMP_NUM_THREADS'))
-            _SESSION = tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=nb_thread,
-                                                        allow_soft_placement=True))
-    return _SESSION
+        session = tf.get_default_session()
+    else:
+        if _SESSION is None:
+            if not os.environ.get('OMP_NUM_THREADS'):
+                config = tf.ConfigProto(allow_soft_placement=True)
+            else:
+                nb_thread = int(os.environ.get('OMP_NUM_THREADS'))
+                config = tf.ConfigProto(intra_op_parallelism_threads=nb_thread,
+                                        allow_soft_placement=True)
+            _SESSION = tf.Session(config=config)
+        session = _SESSION
+    if not _MANUAL_VAR_INIT:
+        _initialize_variables()
+    return session
 
 
 def set_session(session):
@@ -142,30 +173,34 @@ def variable(value, dtype=_FLOATX, name=None):
     '''
     if hasattr(value, 'tocoo'):
         sparse_coo = value.tocoo()
-        indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), np.expand_dims(sparse_coo.col, 1)), 1)
+        indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),
+                                  np.expand_dims(sparse_coo.col, 1)), 1)
         # SparseTensor doesn't need initialization
-        return tf.SparseTensor(indices=indices, values=value.data, shape=value.shape)
-
-    v = tf.Variable(value, dtype=_convert_string_dtype(dtype), name=name)
-    if _MANUAL_VAR_INIT:
+        v = tf.SparseTensor(indices=indices, values=sparse_coo.data, shape=sparse_coo.shape)
+        v._dims = len(sparse_coo.shape)
         return v
-    if tf.get_default_graph() is get_session().graph:
-        try:
-            get_session().run(v.initializer)
-        except tf.errors.InvalidArgumentError:
-            warnings.warn('Could not automatically initialize variable, '
-                          'make sure you do it manually (e.g. via '
-                          '`tf.initialize_all_variables()`).')
-    else:
-        warnings.warn('The default TensorFlow graph is not the graph '
-                      'associated with the TensorFlow session currently '
-                      'registered with Keras, and as such Keras '
-                      'was not able to automatically initialize a variable. '
-                      'You should consider registering the proper session '
-                      'with Keras via `K.set_session(sess)`.')
+    v = tf.Variable(value, dtype=_convert_string_dtype(dtype), name=name)
     return v
 
 
+def _initialize_variables():
+    if hasattr(tf, 'global_variables'):
+        variables = tf.global_variables()
+    else:
+        variables = tf.all_variables()
+
+    uninitialized_variables = []
+    for v in variables:
+        if not hasattr(v, '_keras_initialized') or not v._keras_initialized:
+            uninitialized_variables.append(v)
+            v._keras_initialized = True
+    if uninitialized_variables:
+        sess = get_session()
+        if hasattr(tf, 'variables_initializer'):
+            sess.run(tf.variables_initializer(uninitialized_variables))
+        else:
+            sess.run(tf.initialize_variables(uninitialized_variables))
+
 def placeholder(shape=None, ndim=None, dtype=_FLOATX, sparse=False, name=None):
     '''Instantiates a placeholder.
 
@@ -185,8 +220,8 @@ def placeholder(shape=None, ndim=None, dtype=_FLOATX, sparse=False, name=None):
         if ndim:
             shape = tuple([None for _ in range(ndim)])
     if sparse:
-        tf_shape = tf.constant(np.array(list([0 for _ in range(len(shape))]), dtype=np.int64))
-        x = tf.sparse_placeholder(dtype, shape=tf_shape, name=name)
+        x = tf.sparse_placeholder(dtype, name=name)
+        x._dims = len(shape)
     else:
         x = tf.placeholder(dtype, shape=shape, name=name)
     x._keras_shape = shape
@@ -213,7 +248,7 @@ def ndim(x):
     '''Returns the number of axes in a tensor, as an integer.
     '''
     if is_sparse(x):
-        return int(x.shape.get_shape()[0])
+        return x._dims
 
     dims = x.get_shape()._dims
     if dims is not None:
@@ -239,7 +274,8 @@ def zeros(shape, dtype=_FLOATX, name=None):
     '''
     shape = tuple(map(int, shape))
     tf_dtype = _convert_string_dtype(dtype)
-    return variable(tf.constant_initializer(0., dtype=tf_dtype)(shape), dtype, name)
+    return variable(tf.constant_initializer(0., dtype=tf_dtype)(shape),
+                    dtype, name)
 
 
 def ones(shape, dtype=_FLOATX, name=None):
@@ -247,7 +283,8 @@ def ones(shape, dtype=_FLOATX, name=None):
     '''
     shape = tuple(map(int, shape))
     tf_dtype = _convert_string_dtype(dtype)
-    return variable(tf.constant_initializer(1., dtype=tf_dtype)(shape), dtype, name)
+    return variable(tf.constant_initializer(1., dtype=tf_dtype)(shape),
+                    dtype, name)
 
 
 def eye(size, dtype=_FLOATX, name=None):
@@ -746,14 +783,16 @@ def resize_images(X, height_factor, width_factor, dim_ordering):
         X = permute_dimensions(X, [0, 2, 3, 1])
         X = tf.image.resize_nearest_neighbor(X, new_shape)
         X = permute_dimensions(X, [0, 3, 1, 2])
-        X.set_shape((None, None, original_shape[2] * height_factor, original_shape[3] * width_factor))
+        X.set_shape((None, None, original_shape[2] * height_factor if original_shape[2] is not None else None,
+                    original_shape[3] * width_factor if original_shape[3] is not None else None))
         return X
     elif dim_ordering == 'tf':
         original_shape = int_shape(X)
         new_shape = tf.shape(X)[1:3]
         new_shape *= tf.constant(np.array([height_factor, width_factor]).astype('int32'))
         X = tf.image.resize_nearest_neighbor(X, new_shape)
-        X.set_shape((None, original_shape[1] * height_factor, original_shape[2] * width_factor, None))
+        X.set_shape((None, original_shape[1] * height_factor if original_shape[1] is not None else None,
+                    original_shape[2] * width_factor if original_shape[2] is not None else None, None))
         return X
     else:
         raise Exception('Invalid dim_ordering: ' + dim_ordering)
@@ -844,10 +883,23 @@ def temporal_padding(x, padding=1):
     return tf.pad(x, pattern)
 
 
-def spatial_2d_padding(x, padding=(1, 1), dim_ordering=_IMAGE_DIM_ORDERING):
+def asymmetric_temporal_padding(x, left_pad=1, right_pad=1):
+    '''Pad the middle dimension of a 3D tensor
+    with "left_pad" zeros left and "right_pad" right.
+    '''
+    pattern = [[0, 0], [left_pad, right_pad], [0, 0]]
+    return tf.pad(x, pattern)
+
+
+def spatial_2d_padding(x, padding=(1, 1), dim_ordering='default'):
     '''Pads the 2nd and 3rd dimensions of a 4D tensor
     with "padding[0]" and "padding[1]" (resp.) zeros left and right.
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
+
     if dim_ordering == 'th':
         pattern = [[0, 0], [0, 0],
                    [padding[0], padding[0]], [padding[1], padding[1]]]
@@ -858,13 +910,43 @@ def spatial_2d_padding(x, padding=(1, 1), dim_ordering=_IMAGE_DIM_ORDERING):
     return tf.pad(x, pattern)
 
 
-def spatial_3d_padding(x, padding=(1, 1, 1), dim_ordering=_IMAGE_DIM_ORDERING):
+def asymmetric_spatial_2d_padding(x, top_pad=1, bottom_pad=1,
+                                  left_pad=1, right_pad=1,
+                                  dim_ordering='default'):
+    '''Pad the rows and columns of a 4D tensor
+    with "top_pad", "bottom_pad", "left_pad", "right_pad" (resp.) zeros
+    rows on top, bottom; cols on left, right.
+    '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
+
+    if dim_ordering == 'th':
+        pattern = [[0, 0],
+                   [0, 0],
+                   [top_pad, bottom_pad],
+                   [left_pad, right_pad]]
+    else:
+        pattern = [[0, 0],
+                   [top_pad, bottom_pad],
+                   [left_pad, right_pad],
+                   [0, 0]]
+    return tf.pad(x, pattern)
+
+
+def spatial_3d_padding(x, padding=(1, 1, 1), dim_ordering='default'):
     '''Pads 5D tensor with zeros for the depth, height, width dimension with
     "padding[0]", "padding[1]" and "padding[2]" (resp.) zeros left and right
 
     For 'tf' dim_ordering, the 2nd, 3rd and 4th dimension will be padded.
     For 'th' dim_ordering, the 3rd, 4th and 5th dimension will be padded.
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
+
     if dim_ordering == 'th':
         pattern = [
             [0, 0],
@@ -1006,8 +1088,9 @@ class Function(object):
         for tensor, value in zip(self.inputs, inputs):
             if is_sparse(tensor):
                 sparse_coo = value.tocoo()
-                indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), np.expand_dims(sparse_coo.col, 1)), 1)
-                value = (indices, value.data, value.shape)
+                indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),
+                                          np.expand_dims(sparse_coo.col, 1)), 1)
+                value = (indices, sparse_coo.data, sparse_coo.shape)
             feed_dict[tensor] = value
         session = get_session()
         updated = session.run(self.outputs + [self.updates_op], feed_dict=feed_dict)
@@ -1024,8 +1107,8 @@ def function(inputs, outputs, updates=[], **kwargs):
     '''
     if len(kwargs) > 0:
         msg = [
-            "Expected no kwargs, you passed %s" % len(kwargs),
-            "kwargs passed to function are ignored with Tensorflow backend"
+            'Expected no kwargs, you passed %s' % len(kwargs),
+            'kwargs passed to function are ignored with Tensorflow backend'
         ]
         warnings.warn('\n'.join(msg))
     return Function(inputs, outputs, updates=updates)
@@ -1094,6 +1177,13 @@ def rnn(step_function, inputs, initial_states,
     axes = [1, 0] + list(range(2, ndim))
     inputs = tf.transpose(inputs, (axes))
 
+    if mask is not None:
+        if mask.dtype != tf.bool:
+            mask = tf.cast(mask, tf.bool)
+        if len(mask.get_shape()) == ndim - 1:
+            mask = expand_dims(mask)
+        mask = tf.transpose(mask, axes)
+
     if constants is None:
         constants = []
 
@@ -1110,13 +1200,7 @@ def rnn(step_function, inputs, initial_states,
             input_list.reverse()
 
         if mask is not None:
-            # Transpose not supported by bool tensor types, hence round-trip to uint8.
-            mask = tf.cast(mask, tf.uint8)
-            if len(mask.get_shape()) == ndim - 1:
-                mask = expand_dims(mask)
-            mask = tf.cast(tf.transpose(mask, axes), tf.bool)
             mask_list = tf.unpack(mask)
-
             if go_backwards:
                 mask_list.reverse()
 
@@ -1160,26 +1244,25 @@ def rnn(step_function, inputs, initial_states,
             outputs = tf.pack(successive_outputs)
 
     else:
-        from tensorflow.python.ops.rnn import _dynamic_rnn_loop
-
         if go_backwards:
             inputs = tf.reverse(inputs, [True] + [False] * (ndim - 1))
 
-        states = initial_states
-        nb_states = len(states)
-        if nb_states == 0:
-            # use dummy state, otherwise _dynamic_rnn_loop breaks
-            state = inputs[:, 0, :]
-            state_size = state.get_shape()[-1]
-        else:
-            state_size = int(states[0].get_shape()[-1])
-            if nb_states == 1:
-                state = states[0]
-            else:
-                state = tf.concat(1, states)
+        states = tuple(initial_states)
+
+        time_steps = tf.shape(inputs)[0]
+        output_ta = tensor_array_ops.TensorArray(
+            dtype=inputs.dtype,
+            size=time_steps,
+            tensor_array_name='output_ta')
+        input_ta = tensor_array_ops.TensorArray(
+            dtype=inputs.dtype,
+            size=time_steps,
+            tensor_array_name='input_ta')
+        input_ta = input_ta.unpack(inputs)
+        time = tf.constant(0, dtype='int32', name='time')
 
         if mask is not None:
-            if len(initial_states) == 0:
+            if len(states) == 0:
                 raise ValueError('No initial states provided! '
                                  'When using masking in an RNN, you should '
                                  'provide initial states '
@@ -1189,92 +1272,64 @@ def rnn(step_function, inputs, initial_states,
             if go_backwards:
                 mask = tf.reverse(mask, [True] + [False] * (ndim - 2))
 
-            # Transpose not supported by bool tensor types, hence round-trip to uint8.
-            mask = tf.cast(mask, tf.uint8)
-            if len(mask.get_shape()) == ndim - 1:
-                mask = expand_dims(mask)
-            mask = tf.transpose(mask, axes)
-            inputs = tf.concat(2, [tf.cast(mask, inputs.dtype), inputs])
+            mask_ta = tensor_array_ops.TensorArray(
+                dtype=tf.bool,
+                size=time_steps,
+                tensor_array_name='mask_ta')
+            mask_ta = mask_ta.unpack(mask)
 
-            def _step(input, state):
-                if nb_states > 1:
-                    states = []
-                    for i in range(nb_states):
-                        states.append(state[:, i * state_size: (i + 1) * state_size])
-                else:
-                    states = [state]
-                mask_t = tf.cast(input[:, 0], tf.bool)
-                input = input[:, 1:]
-                output, new_states = step_function(input, states + constants)
-
-                output = tf.select(mask_t, output, states[0])
-                new_states = [tf.select(mask_t, new_states[i], states[i]) for i in range(len(states))]
-
-                if len(new_states) == 1:
-                    new_state = new_states[0]
-                else:
-                    new_state = tf.concat(1, new_states)
-
-                return output, new_state
+            def _step(time, output_ta_t, *states):
+                current_input = input_ta.read(time)
+                mask_t = mask_ta.read(time)
+                output, new_states = step_function(current_input,
+                                                   tuple(states) +
+                                                   tuple(constants))
+                tiled_mask_t = tf.tile(mask_t, tf.pack([1, tf.shape(output)[1]]))
+                output = tf.select(tiled_mask_t, output, states[0])
+                new_states = [tf.select(tiled_mask_t, new_states[i], states[i]) for i in range(len(states))]
+                output_ta_t = output_ta_t.write(time, output)
+                return (time + 1, output_ta_t) + tuple(new_states)
         else:
-            def _step(input, state):
-                if nb_states > 1:
-                    states = []
-                    for i in range(nb_states):
-                        states.append(state[:, i * state_size: (i + 1) * state_size])
-                elif nb_states == 1:
-                    states = [state]
-                else:
-                    states = []
-                output, new_states = step_function(input, states + constants)
+            def _step(time, output_ta_t, *states):
+                current_input = input_ta.read(time)
+                output, new_states = step_function(current_input,
+                                                   tuple(states) +
+                                                   tuple(constants))
+                output_ta_t = output_ta_t.write(time, output)
+                return (time + 1, output_ta_t) + tuple(new_states)
 
-                if len(new_states) > 1:
-                    new_state = tf.concat(1, new_states)
-                elif len(new_states) == 1:
-                    new_state = new_states[0]
-                else:
-                    # return dummy state, otherwise _dynamic_rnn_loop breaks
-                    new_state = output
-                return output, new_state
-
-        _step.state_size = state_size * nb_states
-        # recover output size by calling _step on the first input
-        slice_begin = tf.pack([0] * ndim)
-        slice_size = tf.pack([1] + [-1] * (ndim - 1))
-        first_input = tf.slice(inputs, slice_begin, slice_size)
-        first_input = tf.squeeze(first_input, [0])
-        _step.output_size = int(_step(first_input, state)[0].get_shape()[-1])
-
-        (outputs, final_state) = _dynamic_rnn_loop(
-            _step,
-            inputs,
-            state,
+        final_outputs = control_flow_ops.while_loop(
+            cond=lambda time, *_: time < time_steps,
+            body=_step,
+            loop_vars=(time, output_ta) + states,
             parallel_iterations=32,
-            swap_memory=True,
-            sequence_length=None)
+            swap_memory=True)
+        last_time = final_outputs[0]
+        output_ta = final_outputs[1]
+        new_states = final_outputs[2:]
 
-        if nb_states > 1:
-            new_states = []
-            for i in range(nb_states):
-                new_states.append(final_state[:, i * state_size: (i + 1) * state_size])
-        elif nb_states == 1:
-            new_states = [final_state]
-        else:
-            new_states = []
-
-        # all this circus is to recover the last vector in the sequence.
-        slice_begin = tf.pack([tf.shape(outputs)[0] - 1] + [0] * (ndim - 1))
-        slice_size = tf.pack([1] + [-1] * (ndim - 1))
-        last_output = tf.slice(outputs, slice_begin, slice_size)
-        last_output = tf.squeeze(last_output, [0])
+        outputs = output_ta.pack()
+        last_output = output_ta.read(last_time - 1)
 
     axes = [1, 0] + list(range(2, len(outputs.get_shape())))
     outputs = tf.transpose(outputs, axes)
     return last_output, outputs, new_states
 
 
+def _cond(condition, then_lambda, else_lambda):
+    '''Backwards compatible interface to tf.cond prior to public introduction.
+    '''
+    try:
+        cond_fn = tf.cond
+    except AttributeError:
+        from tensorflow.python.ops import control_flow_ops
+        cond_fn = control_flow_ops.cond
+    return cond_fn(condition, then_lambda, else_lambda)
+
+
 def switch(condition, then_expression, else_expression):
-    '''Switches between two operations depending on a scalar value (int or bool).
+    '''Switches between two operations
+    depending on a scalar value (int or bool).
     Note that both `then_expression` and `else_expression`
     should be symbolic tensors of the *same shape*.
 
@@ -1284,9 +1339,11 @@ def switch(condition, then_expression, else_expression):
         else_expression: TensorFlow operation.
     '''
     x_shape = copy.copy(then_expression.get_shape())
-    x = tf.python.control_flow_ops.cond(tf.cast(condition, 'bool'),
-                                        lambda: then_expression,
-                                        lambda: else_expression)
+    if condition.dtype != tf.bool:
+        condition = tf.cast(condition, 'bool')
+    x = _cond(condition,
+              lambda: then_expression,
+              lambda: else_expression)
     x.set_shape(x_shape)
     return x
 
@@ -1295,17 +1352,13 @@ def in_train_phase(x, alt):
     '''Selects `x` in train phase, and `alt` otherwise.
     Note that `alt` should have the *same shape* as `x`.
     '''
-    if _LEARNING_PHASE is 1:
+    if learning_phase() is 1:
         return x
-    elif _LEARNING_PHASE is 0:
+    elif learning_phase() is 0:
         return alt
-    # else: assume learning phase is a placeholder.
-    x_shape = copy.copy(x.get_shape())
-    x = tf.python.control_flow_ops.cond(tf.cast(_LEARNING_PHASE, 'bool'),
-                                        lambda: x,
-                                        lambda: alt)
+    # else: assume learning phase is a placeholder tensor.
+    x = switch(learning_phase(), x, alt)
     x._uses_learning_phase = True
-    x.set_shape(x_shape)
     return x
 
 
@@ -1313,16 +1366,13 @@ def in_test_phase(x, alt):
     '''Selects `x` in test phase, and `alt` otherwise.
     Note that `alt` should have the *same shape* as `x`.
     '''
-    if _LEARNING_PHASE is 1:
+    if learning_phase() is 1:
         return alt
-    elif _LEARNING_PHASE is 0:
+    elif learning_phase() is 0:
         return x
-    x_shape = copy.copy(x.get_shape())
-    x = tf.python.control_flow_ops.cond(tf.cast(_LEARNING_PHASE, 'bool'),
-                                        lambda: alt,
-                                        lambda: x)
+    # else: assume learning phase is a placeholder tensor.
+    x = switch(learning_phase(), alt, x)
     x._uses_learning_phase = True
-    x.set_shape(x_shape)
     return x
 
 
@@ -1348,6 +1398,20 @@ def relu(x, alpha=0., max_value=None):
     return x
 
 
+def elu(x, alpha=1.):
+    '''Exponential linear unit.
+
+    # Arguments
+        x: Tensor to compute the activation function for.
+        alpha: scalar
+    '''
+    res = tf.nn.elu(x)
+    if alpha == 1:
+        return res
+    else:
+        return tf.select(x > 0, res, alpha * res)
+
+
 def softmax(x):
     '''Softmax of a tensor.
     '''
@@ -1361,6 +1425,8 @@ def softplus(x):
 
 
 def softsign(x):
+    '''Softsign of a tensor.
+    '''
     return tf.nn.softsign(x)
 
 
@@ -1471,6 +1537,21 @@ def l2_normalize(x, axis):
     return tf.nn.l2_normalize(x, dim=axis)
 
 
+def in_top_k(predictions, targets, k):
+    '''Returns whether the `targets` are in the top `k` `predictions`
+
+    # Arguments
+        predictions: A tensor of shape batch_size x classess and type float32.
+        targets: A tensor of shape batch_size and type int32 or int64.
+        k: An int, number of top elements to consider.
+
+    # Returns
+        A tensor of shape batch_size and type bool. output_i is True if
+        targets_i is within top-k values of predictions_i
+    '''
+    return tf.nn.in_top_k(predictions, targets, k)
+
+
 # CONVOLUTIONS
 
 def _preprocess_deconv_output_shape(shape, dim_ordering):
@@ -1555,8 +1636,29 @@ def _postprocess_conv3d_output(x, dim_ordering):
     return x
 
 
+def conv1d(x, kernel, stride=1, border_mode='valid',
+           image_shape=None, filter_shape=None):
+    '''1D convolution.
+
+    # Arguments
+        kernel: kernel tensor.
+        strides: stride integer.
+        border_mode: string, "same" or "valid".
+    '''
+    # pre-process dtype
+    if _FLOATX == 'float64':
+        x = tf.cast(x, 'float32')
+        kernel = tf.cast(kernel, 'float32')
+    padding = _preprocess_border_mode(border_mode)
+    x = tf.nn.conv1d(x, kernel, stride, padding=padding)
+    # post-process dtype
+    if _FLOATX == 'float64':
+        x = tf.cast(x, 'float64')
+    return x
+
+
 def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
-           dim_ordering=_IMAGE_DIM_ORDERING,
+           dim_ordering='default',
            image_shape=None, filter_shape=None, filter_dilation=(1, 1)):
     '''2D convolution.
 
@@ -1568,8 +1670,10 @@ def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
             Whether to use Theano or TensorFlow dimension ordering
             for inputs/kernels/ouputs.
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
     if dim_ordering not in {'th', 'tf'}:
-        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
 
     x = _preprocess_conv2d_input(x, dim_ordering)
     kernel = _preprocess_conv2d_kernel(kernel, dim_ordering)
@@ -1586,7 +1690,7 @@ def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
 
 def deconv2d(x, kernel, output_shape, strides=(1, 1),
              border_mode='valid',
-             dim_ordering=_IMAGE_DIM_ORDERING,
+             dim_ordering='default',
              image_shape=None, filter_shape=None):
     '''2D deconvolution (i.e. transposed convolution).
 
@@ -1600,8 +1704,10 @@ def deconv2d(x, kernel, output_shape, strides=(1, 1),
             Whether to use Theano or TensorFlow dimension ordering
             for inputs/kernels/ouputs.
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
     if dim_ordering not in {'th', 'tf'}:
-        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
 
     x = _preprocess_conv2d_input(x, dim_ordering)
     output_shape = _preprocess_deconv_output_shape(output_shape, dim_ordering)
@@ -1617,10 +1723,12 @@ def deconv2d(x, kernel, output_shape, strides=(1, 1),
 
 def atrous_conv2d(x, kernel, rate=1,
                   border_mode='valid',
-                  dim_ordering=_IMAGE_DIM_ORDERING,
+                  dim_ordering='default',
                   image_shape=None, filter_shape=None):
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
     if dim_ordering not in {'th', 'tf'}:
-        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
     if rate == 1:
         return conv2d(x, kernel, strides=(1, 1), border_mode=border_mode,
                       dim_ordering=dim_ordering)
@@ -1634,9 +1742,11 @@ def atrous_conv2d(x, kernel, rate=1,
 
 
 def separable_conv2d(x, depthwise_kernel, pointwise_kernel, strides=(1, 1),
-                     border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING):
+                     border_mode='valid', dim_ordering='default'):
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
     if dim_ordering not in {'th', 'tf'}:
-        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
 
     x = _preprocess_conv2d_input(x, dim_ordering)
     depthwise_kernel = _preprocess_conv2d_kernel(depthwise_kernel,
@@ -1652,7 +1762,7 @@ def separable_conv2d(x, depthwise_kernel, pointwise_kernel, strides=(1, 1),
 
 
 def conv3d(x, kernel, strides=(1, 1, 1),
-           border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING,
+           border_mode='valid', dim_ordering='default',
            volume_shape=None, filter_shape=None):
     '''3D convolution.
 
@@ -1664,8 +1774,10 @@ def conv3d(x, kernel, strides=(1, 1, 1),
             Whether to use Theano or TensorFlow dimension ordering
             for inputs/kernels/ouputs.
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
     if dim_ordering not in {'th', 'tf'}:
-        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
 
     x = _preprocess_conv3d_input(x, dim_ordering)
     kernel = _preprocess_conv3d_kernel(kernel, dim_ordering)
@@ -1677,7 +1789,7 @@ def conv3d(x, kernel, strides=(1, 1, 1),
 
 
 def pool2d(x, pool_size, strides=(1, 1),
-           border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING,
+           border_mode='valid', dim_ordering='default',
            pool_mode='max'):
     '''2D Pooling.
 
@@ -1688,8 +1800,10 @@ def pool2d(x, pool_size, strides=(1, 1),
         dim_ordering: one of "th", "tf".
         pool_mode: one of "max", "avg".
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
     if dim_ordering not in {'th', 'tf'}:
-        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
 
     padding = _preprocess_border_mode(border_mode)
     strides = (1,) + strides + (1,)
@@ -1708,7 +1822,7 @@ def pool2d(x, pool_size, strides=(1, 1),
 
 
 def pool3d(x, pool_size, strides=(1, 1, 1), border_mode='valid',
-           dim_ordering=_IMAGE_DIM_ORDERING, pool_mode='max'):
+           dim_ordering='default', pool_mode='max'):
     '''3D Pooling.
 
     # Arguments
@@ -1718,8 +1832,10 @@ def pool3d(x, pool_size, strides=(1, 1, 1), border_mode='valid',
         dim_ordering: one of "th", "tf".
         pool_mode: one of "max", "avg".
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
     if dim_ordering not in {'th', 'tf'}:
-        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
 
     padding = _preprocess_border_mode(border_mode)
     strides = (1,) + strides + (1,)
@@ -1774,9 +1890,9 @@ def ctc_label_dense_to_sparse(labels, label_lengths):
     max_num_labels_tns = tf.pack([label_shape[1]])
 
     def range_less_than(previous_state, current_input):
-        return tf.expand_dims(tf.range(label_shape[1]), 0) < current_input
+        return tf.expand_dims(tf.range(label_shape[1]), 0) < tf.fill(max_num_labels_tns, current_input)
 
-    init = tf.cast(tf.fill(max_num_labels_tns, 0), tf.bool)
+    init = tf.cast(tf.fill([1, label_shape[1]], 0), tf.bool)
     dense_mask = functional_ops.scan(range_less_than, label_lengths,
                                      initializer=init, parallel_iterations=1)
     dense_mask = dense_mask[:, 0, :]
@@ -1864,3 +1980,52 @@ def ctc_decode(y_pred, input_length, greedy=True, beam_width=100,
                      for st in decoded]
 
     return (decoded_dense, log_prob)
+
+
+# HIGH ORDER FUNCTIONS
+
+def map_fn(fn, elems, name=None):
+    '''Map the function fn over the elements elems and return the outputs.
+
+    # Arguments
+        fn: Callable that will be called upon each element in elems
+        elems: tensor
+        name: A string name for the map node in the graph
+
+    # Returns
+        Tensor with first dimension equal to the elems and second depending on
+        fn
+    '''
+    return tf.map_fn(fn, elems, name=name)
+
+
+def foldl(fn, elems, initializer=None, name=None):
+    '''Reduce elems using fn to combine them from left to right.
+
+    # Arguments
+        fn: Callable that will be called upon each element in elems and an
+            accumulator, for instance lambda acc, x: acc + x
+        elems: tensor
+        initializer: The first value used (elems[0] in case of None)
+        name: A string name for the foldl node in the graph
+
+    # Returns
+        Same type and shape as initializer
+    '''
+    return tf.foldl(fn, elems, initializer=initializer, name=name)
+
+
+def foldr(fn, elems, initializer=None, name=None):
+    '''Reduce elems using fn to combine them from right to left.
+
+    # Arguments
+        fn: Callable that will be called upon each element in elems and an
+            accumulator, for instance lambda acc, x: acc + x
+        elems: tensor
+        initializer: The first value used (elems[-1] in case of None)
+        name: A string name for the foldr node in the graph
+
+    # Returns
+        Same type and shape as initializer
+    '''
+    return tf.foldr(fn, elems, initializer=initializer, name=name)

keras/backend/theano_backend.py

@@ -14,7 +14,7 @@ except ImportError:
     from theano.sandbox.softsign import softsign as T_softsign
 import inspect
 import numpy as np
-from .common import _FLOATX, _EPSILON, _IMAGE_DIM_ORDERING
+from .common import _FLOATX, _EPSILON, image_dim_ordering
 py_all = all
 
 
@@ -35,6 +35,7 @@ def set_learning_phase(value):
                          '0 or 1.')
     _LEARNING_PHASE = value
 
+
 # VARIABLE MANIPULATION
 
 
@@ -88,7 +89,7 @@ def placeholder(shape=None, ndim=None, dtype=_FLOATX, sparse=False, name=None):
 
 
 def shape(x):
-    '''Return the shape of a tensor.
+    '''Returns the shape of a tensor.
 
     Warning: type returned will be different for
     Theano backend (Theano tensor type) and TF backend (TF TensorShape).
@@ -105,25 +106,25 @@ def dtype(x):
 
 
 def eval(x):
-    '''Run a graph.
+    '''Returns the value of a tensor.
     '''
     return to_dense(x).eval()
 
 
 def zeros(shape, dtype=_FLOATX, name=None):
-    '''Instantiate an all-zeros variable.
+    '''Instantiates an all-zeros variable.
     '''
     return variable(np.zeros(shape), dtype, name)
 
 
 def ones(shape, dtype=_FLOATX, name=None):
-    '''Instantiate an all-ones variable.
+    '''Instantiates an all-ones variable.
     '''
     return variable(np.ones(shape), dtype, name)
 
 
 def eye(size, dtype=_FLOATX, name=None):
-    '''Instantiate an identity matrix.
+    '''Instantiates an identity matrix.
     '''
     return variable(np.eye(size), dtype, name)
 
@@ -147,7 +148,7 @@ def random_normal_variable(shape, mean, scale, dtype=_FLOATX, name=None):
 
 
 def count_params(x):
-    '''Return number of scalars in a tensor.
+    '''Returns the number of scalars in a tensor.
 
     Return: numpy integer.
     '''
@@ -393,8 +394,21 @@ def cos(x):
 
 def normalize_batch_in_training(x, gamma, beta,
                                 reduction_axes, epsilon=0.0001):
-    '''Compute mean and std for batch then apply batch_normalization on batch.
+    '''Computes mean and std for batch then apply batch_normalization on batch.
     '''
+    dev = theano.config.device
+    use_cudnn = ndim(x) < 5 and reduction_axes == [0, 2, 3] and (dev.startswith('cuda') or dev.startswith('gpu'))
+    if use_cudnn:
+        broadcast_beta = beta.dimshuffle('x', 0, 'x', 'x')
+        broadcast_gamma = gamma.dimshuffle('x', 0, 'x', 'x')
+        try:
+            normed, mean, stdinv = theano.sandbox.cuda.dnn.dnn_batch_normalization_train(
+                x, broadcast_gamma, broadcast_beta, 'spatial', epsilon)
+            var = T.inv(stdinv ** 2)
+            return normed, T.flatten(mean), T.flatten(var)
+        except AttributeError:
+            pass
+
     var = x.var(reduction_axes)
     mean = x.mean(reduction_axes)
 
@@ -424,10 +438,25 @@ def batch_normalization(x, mean, var, beta, gamma, epsilon=0.0001):
     use_cudnn = ndim < 5 and (dev.startswith('cuda') or dev.startswith('gpu'))
     if use_cudnn:
         try:
-            return theano.sandbox.cuda.dnn.dnn_batch_normalization_test(x, gamma, beta, mean, var,
-                                                                        'spatial', epsilon)
+            axis = mean.broadcastable.index(False)
+            if axis != 1:
+                shuffle_pattern = list(range(ndim))
+                shuffle_pattern[1] = shuffle_pattern[axis]
+                shuffle_pattern[axis] = 1
+                x = x.dimshuffle(shuffle_pattern)
+                mean = mean.dimshuffle(shuffle_pattern)
+                var = var.dimshuffle(shuffle_pattern)
+                beta = beta.dimshuffle(shuffle_pattern)
+                gamma = gamma.dimshuffle(shuffle_pattern)
+            normed = theano.sandbox.cuda.dnn.dnn_batch_normalization_test(x, gamma, beta, mean, var,
+                                                                          'spatial', epsilon)
+            if axis != 1:
+                normed = normed.dimshuffle(shuffle_pattern)
+            return normed
         except AttributeError:
             pass
+        except ValueError:
+            pass
     return T.nnet.bn.batch_normalization(x, gamma, beta, mean, sqrt(var + epsilon),
                                          mode='high_mem')
 
@@ -573,10 +602,30 @@ def temporal_padding(x, padding=1):
     return T.set_subtensor(output[:, padding:x.shape[1] + padding, :], x)
 
 
-def spatial_2d_padding(x, padding=(1, 1), dim_ordering=_IMAGE_DIM_ORDERING):
+def asymmetric_temporal_padding(x, left_pad=1, right_pad=1):
+    '''Pad the middle dimension of a 3D tensor
+    with "left_pad" zeros left and "right_pad" right.
+
+    Apologies for the inane API, but Theano makes this
+    really hard.
+    '''
+    input_shape = x.shape
+    output_shape = (input_shape[0],
+                    input_shape[1] + left_pad + right_pad,
+                    input_shape[2])
+    output = T.zeros(output_shape)
+    return T.set_subtensor(output[:, left_pad:x.shape[1] + left_pad, :], x)
+
+
+def spatial_2d_padding(x, padding=(1, 1), dim_ordering='default'):
     '''Pad the 2nd and 3rd dimensions of a 4D tensor
     with "padding[0]" and "padding[1]" (resp.) zeros left and right.
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
+
     input_shape = x.shape
     if dim_ordering == 'th':
         output_shape = (input_shape[0],
@@ -604,10 +653,55 @@ def spatial_2d_padding(x, padding=(1, 1), dim_ordering=_IMAGE_DIM_ORDERING):
     return T.set_subtensor(output[indices], x)
 
 
-def spatial_3d_padding(x, padding=(1, 1, 1), dim_ordering=_IMAGE_DIM_ORDERING):
+def asymmetric_spatial_2d_padding(x, top_pad=1, bottom_pad=1,
+                                  left_pad=1, right_pad=1,
+                                  dim_ordering='default'):
+    '''Pad the rows and columns of a 4D tensor
+    with "top_pad", "bottom_pad", "left_pad", "right_pad" (resp.) zeros
+    rows on top, bottom; cols on left, right.
+    '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
+
+    input_shape = x.shape
+    if dim_ordering == 'th':
+        output_shape = (input_shape[0],
+                        input_shape[1],
+                        input_shape[2] + top_pad + bottom_pad,
+                        input_shape[3] + left_pad + right_pad)
+        output = T.zeros(output_shape)
+        indices = (slice(None),
+                   slice(None),
+                   slice(top_pad, input_shape[2] + top_pad),
+                   slice(left_pad, input_shape[3] + left_pad))
+
+    elif dim_ordering == 'tf':
+        output_shape = (input_shape[0],
+                        input_shape[1] + top_pad + bottom_pad,
+                        input_shape[2] + left_pad + right_pad,
+                        input_shape[3])
+        print(output_shape)
+        output = T.zeros(output_shape)
+        indices = (slice(None),
+                   slice(top_pad, input_shape[1] + top_pad),
+                   slice(left_pad, input_shape[2] + left_pad),
+                   slice(None))
+    else:
+        raise Exception('Invalid dim_ordering: ' + dim_ordering)
+    return T.set_subtensor(output[indices], x)
+
+
+def spatial_3d_padding(x, padding=(1, 1, 1), dim_ordering='default'):
     '''Pad the 2nd, 3rd and 4th dimensions of a 5D tensor
     with "padding[0]", "padding[1]" and "padding[2]" (resp.) zeros left and right.
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
+
     input_shape = x.shape
     if dim_ordering == 'th':
         output_shape = (input_shape[0],
@@ -931,11 +1025,26 @@ def in_test_phase(x, alt):
 
 # NN OPERATIONS
 
+def _assert_has_capability(module, func):
+    assert hasattr(module, func), ('It looks like like your version of '
+                                   'Theano is out of date. '
+                                   'Install the latest version with:\n'
+                                   'pip install git+git://github.com/Theano/Theano.git --upgrade --no-deps')
+
+
+def elu(x, alpha=1.0):
+    """ Exponential linear unit
+
+    # Arguments
+        x: Tensor to compute the activation function for.
+        alpha: scalar
+    """
+    _assert_has_capability(T.nnet, 'elu')
+    return T.nnet.elu(x, alpha)
+
+
 def relu(x, alpha=0., max_value=None):
-    assert hasattr(T.nnet, 'relu'), ('It looks like like your version of '
-                                     'Theano is out of date. '
-                                     'Install the latest version with:\n'
-                                     'pip install git+git://github.com/Theano/Theano.git --upgrade --no-deps')
+    _assert_has_capability(T.nnet, 'relu')
     x = T.nnet.relu(x, alpha)
     if max_value is not None:
         x = T.minimum(x, max_value)
@@ -1028,6 +1137,23 @@ def l2_normalize(x, axis):
     return x / norm
 
 
+def in_top_k(predictions, targets, k):
+    '''Returns whether the `targets` are in the top `k` `predictions`
+
+    # Arguments
+        predictions: A tensor of shape batch_size x classess and type float32.
+        targets: A tensor of shape batch_size and type int32 or int64.
+        k: An int, number of top elements to consider.
+
+    # Returns
+        A tensor of shape batch_size and type int. output_i is 1 if
+        targets_i is within top-k values of predictions_i
+    '''
+    predictions_top_k = T.argsort(predictions)[:, -k:]
+    result, _ = theano.map(lambda prediction, target: any(equal(prediction, target)), sequences=[predictions_top_k, targets])
+    return result
+
+
 # CONVOLUTIONS
 
 def _preprocess_conv2d_input(x, dim_ordering):
@@ -1040,6 +1166,16 @@ def _preprocess_conv2d_input(x, dim_ordering):
     return x
 
 
+def _preprocess_conv3d_input(x, dim_ordering):
+    if dim_ordering == 'tf':
+        # TF uses the last dimension as channel dimension,
+        # instead of the 2nd one.
+        # TH input shape: (samples, input_depth, rows, cols, slices)
+        # TF input shape: (samples, rows, cols, slices, input_depth)
+        x = x.dimshuffle((0, 4, 1, 2, 3))
+    return x
+
+
 def _preprocess_conv2d_kernel(kernel, dim_ordering):
     if dim_ordering == 'tf':
         # TF uses the last dimension as channel dimension,
@@ -1050,17 +1186,29 @@ def _preprocess_conv2d_kernel(kernel, dim_ordering):
     return kernel
 
 
+def _preprocess_conv3d_kernel(kernel, dim_ordering):
+    if dim_ordering == 'tf':
+        # TF uses the last dimension as channel dimension,
+        # instead of the 2nd one.
+        # TH kernel shape: (depth, input_depth, rows, cols, slices)
+        # TF kernel shape: (rows, cols, slices, input_depth, depth)
+        kernel = kernel.dimshuffle((4, 3, 0, 1, 2))
+    return kernel
+
+
 def _preprocess_border_mode(border_mode):
     if border_mode == 'same':
         th_border_mode = 'half'
     elif border_mode == 'valid':
         th_border_mode = 'valid'
+    elif border_mode == 'full':
+        th_border_mode = 'full'
     else:
         raise Exception('Border mode not supported: ' + str(border_mode))
     return th_border_mode
 
 
-def _preprocess_image_shape(dim_ordering, image_shape):
+def _preprocess_conv2d_image_shape(dim_ordering, image_shape):
     # Theano might not accept long type
     def int_or_none(value):
         try:
@@ -1076,7 +1224,23 @@ def _preprocess_image_shape(dim_ordering, image_shape):
     return image_shape
 
 
-def _preprocess_filter_shape(dim_ordering, filter_shape):
+def _preprocess_conv3d_volume_shape(dim_ordering, volume_shape):
+    # Theano might not accept long type
+    def int_or_none(value):
+        try:
+            return int(value)
+        except TypeError:
+            return None
+    if dim_ordering == 'tf':
+        if volume_shape:
+            volume_shape = (volume_shape[0], volume_shape[4],
+                            volume_shape[1], volume_shape[2], volume_shape[3])
+    if volume_shape is not None:
+        volume_shape = tuple(int_or_none(v) for v in volume_shape)
+    return volume_shape
+
+
+def _preprocess_conv2d_filter_shape(dim_ordering, filter_shape):
     # Theano might not accept long type
     def int_or_none(value):
         try:
@@ -1092,6 +1256,22 @@ def _preprocess_filter_shape(dim_ordering, filter_shape):
     return filter_shape
 
 
+def _preprocess_conv3d_filter_shape(dim_ordering, filter_shape):
+    # Theano might not accept long type
+    def int_or_none(value):
+        try:
+            return int(value)
+        except TypeError:
+            return None
+    if dim_ordering == 'tf':
+        if filter_shape:
+            filter_shape = (filter_shape[4], filter_shape[3],
+                            filter_shape[0], filter_shape[1], filter_shape[2])
+    if filter_shape is not None:
+        filter_shape = tuple(int_or_none(v) for v in filter_shape)
+    return filter_shape
+
+
 def _postprocess_conv2d_output(conv_out, x, border_mode, np_kernel, strides, dim_ordering):
     if border_mode == 'same':
         if np_kernel.shape[2] % 2 == 0:
@@ -1103,8 +1283,33 @@ def _postprocess_conv2d_output(conv_out, x, border_mode, np_kernel, strides, dim
     return conv_out
 
 
+def _postprocess_conv3d_output(conv_out, x, border_mode, np_kernel, strides, dim_ordering):
+    if border_mode == 'same':
+        if np_kernel.shape[2] % 2 == 0:
+            conv_out = conv_out[:, :, :(x.shape[2] + strides[0] - 1) // strides[0], :, :]
+        if np_kernel.shape[3] % 2 == 0:
+            conv_out = conv_out[:, :, :, :(x.shape[3] + strides[1] - 1) // strides[1], :]
+        if np_kernel.shape[4] % 2 == 0:
+            conv_out = conv_out[:, :, :, :, :(x.shape[4] + strides[2] - 1) // strides[2]]
+    if dim_ordering == 'tf':
+        conv_out = conv_out.dimshuffle((0, 2, 3, 4, 1))
+    return conv_out
+
+
+def conv1d(x, kernel, stride=1, border_mode='valid',
+           image_shape=None, filter_shape=None):
+    '''1D convolution.
+
+    # Arguments
+        kernel: kernel tensor.
+        strides: stride integer.
+        border_mode: string, "same" or "valid".
+    '''
+    raise NotImplementedError
+
+
 def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
-           dim_ordering=_IMAGE_DIM_ORDERING, image_shape=None,
+           dim_ordering='default', image_shape=None,
            filter_shape=None, filter_dilation=(1, 1)):
     '''2D convolution.
 
@@ -1116,6 +1321,8 @@ def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
             Whether to use Theano or TensorFlow dimension ordering
         in inputs/kernels/ouputs.
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
     if dim_ordering not in {'th', 'tf'}:
         raise Exception('Unknown dim_ordering ' + str(dim_ordering))
 
@@ -1123,8 +1330,8 @@ def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
     kernel = _preprocess_conv2d_kernel(kernel, dim_ordering)
     th_border_mode = _preprocess_border_mode(border_mode)
     np_kernel = kernel.eval()
-    image_shape = _preprocess_image_shape(dim_ordering, image_shape)
-    filter_shape = _preprocess_filter_shape(dim_ordering, filter_shape)
+    image_shape = _preprocess_conv2d_image_shape(dim_ordering, image_shape)
+    filter_shape = _preprocess_conv2d_filter_shape(dim_ordering, filter_shape)
 
     # TODO: remove the if statement when theano with no filter dilation is deprecated.
     if filter_dilation == (1, 1):
@@ -1148,7 +1355,7 @@ def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
 
 def deconv2d(x, kernel, output_shape, strides=(1, 1),
              border_mode='valid',
-             dim_ordering=_IMAGE_DIM_ORDERING,
+             dim_ordering='default',
              image_shape=None, filter_shape=None):
     '''2D deconvolution (transposed convolution).
 
@@ -1162,6 +1369,8 @@ def deconv2d(x, kernel, output_shape, strides=(1, 1),
         in inputs/kernels/ouputs.
     '''
     flip_filters = False
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
     if dim_ordering not in {'th', 'tf'}:
         raise Exception('Unknown dim_ordering ' + str(dim_ordering))
 
@@ -1170,7 +1379,7 @@ def deconv2d(x, kernel, output_shape, strides=(1, 1),
     kernel = kernel.dimshuffle((1, 0, 2, 3))
     th_border_mode = _preprocess_border_mode(border_mode)
     np_kernel = kernel.eval()
-    filter_shape = _preprocess_filter_shape(dim_ordering, filter_shape)
+    filter_shape = _preprocess_conv2d_filter_shape(dim_ordering, filter_shape)
 
     op = T.nnet.abstract_conv.AbstractConv2d_gradInputs(imshp=output_shape,
                                                         kshp=filter_shape,
@@ -1186,23 +1395,73 @@ def deconv2d(x, kernel, output_shape, strides=(1, 1),
 
 def atrous_conv2d(x, kernel, rate=1,
                   border_mode='valid',
-                  dim_ordering=_IMAGE_DIM_ORDERING,
+                  dim_ordering='default',
                   image_shape=None, filter_shape=None):
     raise NotImplementedError
 
 
 def separable_conv2d(x, depthwise_kernel, pointwise_kernel, strides=(1, 1),
-                     border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING):
+                     border_mode='valid', dim_ordering='default'):
     raise NotImplementedError
 
 
 def conv3d(x, kernel, strides=(1, 1, 1),
-           border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING,
-           volume_shape=None, filter_shape=None):
+           border_mode='valid', dim_ordering='default',
+           volume_shape=None, filter_shape=None,
+           filter_dilation=(1, 1, 1)):
+    '''3D convolution.
+
+    # Arguments
+        kernel: kernel tensor.
+        strides: strides tuple.
+        border_mode: string, "same" or "valid".
+        dim_ordering: "tf" or "th".
+            Whether to use Theano or TensorFlow dimension ordering
+        in inputs/kernels/ouputs.
+    '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+
+    # TODO: remove this if statement when Theano without AbstractConv3d is deprecated
+    if not hasattr(T.nnet, 'conv3d'):
+        if filter_dilation != (1, 1, 1):
+            raise Exception('conv3d with filter dilation requires Theano '
+                            '0.9.0dev3 or newer.')
+
+        return _old_theano_conv3d(x, kernel, strides, border_mode,
+                                  dim_ordering, volume_shape, filter_shape)
+
+    x = _preprocess_conv3d_input(x, dim_ordering)
+    kernel = _preprocess_conv3d_kernel(kernel, dim_ordering)
+    th_border_mode = _preprocess_border_mode(border_mode)
+    np_kernel = kernel.eval()
+    volume_shape = _preprocess_conv3d_volume_shape(dim_ordering, volume_shape)
+    filter_shape = _preprocess_conv3d_filter_shape(dim_ordering, filter_shape)
+
+    conv_out = T.nnet.conv3d(x, kernel,
+                             border_mode=th_border_mode,
+                             subsample=strides,
+                             input_shape=volume_shape,
+                             filter_shape=filter_shape,
+                             filter_dilation=filter_dilation)
+
+    conv_out = _postprocess_conv3d_output(conv_out, x, border_mode, np_kernel,
+                                          strides, dim_ordering)
+    return conv_out
+
+
+# TODO: remove this function when theano without AbstractConv3d is deprecated
+def _old_theano_conv3d(x, kernel, strides=(1, 1, 1),
+                       border_mode='valid', dim_ordering='default',
+                       volume_shape=None, filter_shape=None):
     '''
     Run on cuDNN if available.
     border_mode: string, "same" or "valid".
     '''
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
     if dim_ordering not in {'th', 'tf'}:
         raise Exception('Unknown dim_ordering ' + str(dim_ordering))
 
@@ -1259,7 +1518,12 @@ def conv3d(x, kernel, strides=(1, 1, 1),
 
 
 def pool2d(x, pool_size, strides=(1, 1), border_mode='valid',
-           dim_ordering=_IMAGE_DIM_ORDERING, pool_mode='max'):
+           dim_ordering='default', pool_mode='max'):
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+
     if border_mode == 'same':
         w_pad = pool_size[0] - 2 if pool_size[0] % 2 == 1 else pool_size[0] - 1
         h_pad = pool_size[1] - 2 if pool_size[1] % 2 == 1 else pool_size[1] - 1
@@ -1276,15 +1540,33 @@ def pool2d(x, pool_size, strides=(1, 1), border_mode='valid',
         x = x.dimshuffle((0, 3, 1, 2))
 
     if pool_mode == 'max':
-        pool_out = pool.pool_2d(x, ds=pool_size, st=strides,
-                                ignore_border=True,
-                                padding=padding,
-                                mode='max')
+        # TODO remove the old call once Theano older than 0.9.0dev4 is deprecated
+        try:
+            # new interface (introduced in 0.9.0dev4)
+            pool_out = pool.pool_2d(x, ws=pool_size, stride=strides,
+                                    ignore_border=True,
+                                    pad=padding,
+                                    mode='max')
+        except TypeError:
+            # old interface
+            pool_out = pool.pool_2d(x, ds=pool_size, st=strides,
+                                    ignore_border=True,
+                                    padding=padding,
+                                    mode='max')
     elif pool_mode == 'avg':
-        pool_out = pool.pool_2d(x, ds=pool_size, st=strides,
-                                ignore_border=True,
-                                padding=padding,
-                                mode='average_exc_pad')
+        # TODO remove the old call once Theano older than 0.9.0dev4 is deprecated
+        try:
+            # new interface (introduced in 0.9.0dev4)
+            pool_out = pool.pool_2d(x, ws=pool_size, stride=strides,
+                                    ignore_border=True,
+                                    pad=padding,
+                                    mode='average_exc_pad')
+        except TypeError:
+            # old interface
+            pool_out = pool.pool_2d(x, ds=pool_size, st=strides,
+                                    ignore_border=True,
+                                    padding=padding,
+                                    mode='average_exc_pad')
     else:
         raise Exception('Invalid pooling mode: ' + str(pool_mode))
 
@@ -1302,7 +1584,89 @@ def pool2d(x, pool_size, strides=(1, 1), border_mode='valid',
 
 
 def pool3d(x, pool_size, strides=(1, 1, 1), border_mode='valid',
-           dim_ordering=_IMAGE_DIM_ORDERING, pool_mode='max'):
+           dim_ordering='default', pool_mode='max'):
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+
+    # TODO: remove this if statement when Theano without pool_3d is deprecated
+    #       (pool_3d was introduced after 0.9.0dev3)
+    if not hasattr(T.signal.pool, 'pool_3d'):
+        return _old_theano_pool3d(x, pool_size, strides, border_mode,
+                                  dim_ordering, pool_mode)
+
+    if border_mode == 'same':
+        w_pad = pool_size[0] - 2 if pool_size[0] % 2 == 1 else pool_size[0] - 1
+        h_pad = pool_size[1] - 2 if pool_size[1] % 2 == 1 else pool_size[1] - 1
+        d_pad = pool_size[2] - 2 if pool_size[2] % 2 == 1 else pool_size[2] - 1
+        padding = (w_pad, h_pad, d_pad)
+    elif border_mode == 'valid':
+        padding = (0, 0, 0)
+    else:
+        raise Exception('Invalid border mode: ' + str(border_mode))
+
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+
+    if dim_ordering == 'tf':
+        x = x.dimshuffle((0, 4, 1, 2, 3))
+
+    if pool_mode == 'max':
+        # TODO remove the old call once Theano older than 0.9.0dev4 is deprecated
+        try:
+            # new interface (introduced in 0.9.0dev4)
+            pool_out = pool.pool_3d(x, ws=pool_size, stride=strides,
+                                    ignore_border=True,
+                                    pad=padding,
+                                    mode='max')
+        except TypeError:
+            # old interface
+            pool_out = pool.pool_3d(x, ds=pool_size, st=strides,
+                                    ignore_border=True,
+                                    padding=padding,
+                                    mode='max')
+    elif pool_mode == 'avg':
+        # TODO remove the old call once Theano older than 0.9.0dev4 is deprecated
+        try:
+            # new interface (introduced in 0.9.0dev4)
+            pool_out = pool.pool_3d(x, ws=pool_size, stride=strides,
+                                    ignore_border=True,
+                                    pad=padding,
+                                    mode='average_exc_pad')
+        except TypeError:
+            # old interface
+            pool_out = pool.pool_3d(x, ds=pool_size, st=strides,
+                                    ignore_border=True,
+                                    padding=padding,
+                                    mode='average_exc_pad')
+    else:
+        raise Exception('Invalid pooling mode: ' + str(pool_mode))
+
+    if border_mode == 'same':
+        expected_width = (x.shape[2] + strides[0] - 1) // strides[0]
+        expected_height = (x.shape[3] + strides[1] - 1) // strides[1]
+        expected_depth = (x.shape[4] + strides[2] - 1) // strides[2]
+
+        pool_out = pool_out[:, :,
+                            : expected_width,
+                            : expected_height,
+                            : expected_depth]
+
+    if dim_ordering == 'tf':
+        pool_out = pool_out.dimshuffle((0, 2, 3, 4, 1))
+    return pool_out
+
+
+# TODO: remove this function when Theano without pool_3d is deprecated
+#       (pool_3d was introduced after 0.9.0dev3)
+def _old_theano_pool3d(x, pool_size, strides=(1, 1, 1), border_mode='valid',
+                       dim_ordering='default', pool_mode='max'):
+    if dim_ordering == 'default':
+        dim_ordering = image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+
     if border_mode == 'same':
         # TODO: add implementation for border_mode="same"
         raise Exception('border_mode="same" not supported with Theano.')
@@ -1393,11 +1757,13 @@ def ctc_interleave_blanks(Y):
     Y_ = T.set_subtensor(Y_[T.arange(Y.shape[0]) * 2 + 1], Y)
     return Y_
 
+
 def ctc_create_skip_idxs(Y):
     skip_idxs = T.arange((Y.shape[0] - 3) // 2) * 2 + 1
     non_repeats = T.neq(Y[skip_idxs], Y[skip_idxs + 2])
     return skip_idxs[non_repeats.nonzero()]
 
+
 def ctc_update_log_p(skip_idxs, zeros, active, log_p_curr, log_p_prev):
     active_skip_idxs = skip_idxs[(skip_idxs < active).nonzero()]
     active_next = T.cast(T.minimum(
@@ -1423,11 +1789,11 @@ def ctc_update_log_p(skip_idxs, zeros, active, log_p_curr, log_p_prev):
     )
     return active_next, log_p_next
 
+
 def ctc_path_probs(predict, Y, alpha=1e-4):
     smoothed_predict = (1 - alpha) * predict[:, Y] + alpha * np.float32(1.) / Y.shape[0]
     L = T.log(smoothed_predict)
     zeros = T.zeros_like(L[0])
-    base = T.set_subtensor(zeros[:1], np.float32(1))
     log_first = zeros
 
     f_skip_idxs = ctc_create_skip_idxs(Y)
@@ -1446,12 +1812,14 @@ def ctc_path_probs(predict, Y, alpha=1e-4):
     log_probs = log_f_probs + log_b_probs[::-1, ::-1] - L
     return log_probs, mask
 
+
 def ctc_cost(predict, Y):
     log_probs, mask = ctc_path_probs(predict, ctc_interleave_blanks(Y))
     common_factor = T.max(log_probs)
     total_log_prob = T.log(T.sum(T.exp(log_probs - common_factor)[mask.nonzero()])) + common_factor
     return -total_log_prob
 
+
 # batchifies original CTC code
 def ctc_batch_cost(y_true, y_pred, input_length, label_length):
     '''Runs CTC loss algorithm on each batch element.
@@ -1476,10 +1844,75 @@ def ctc_batch_cost(y_true, y_pred, input_length, label_length):
         return ctc_cost(y_pred_step, y_true_step)
 
     ret, _ = theano.scan(
-        fn = ctc_step,
+        fn=ctc_step,
         outputs_info=None,
         sequences=[y_true, y_pred, input_length, label_length]
     )
 
     ret = ret.dimshuffle('x', 0)
     return ret
+
+
+# HIGH ORDER FUNCTIONS
+
+def map_fn(fn, elems, name=None):
+    '''Map the function fn over the elements elems and return the outputs.
+
+    # Arguments
+        fn: Callable that will be called upon each element in elems
+        elems: tensor, at least 2 dimensional
+        name: A string name for the map node in the graph
+
+    # Returns
+        Tensor with first dimension equal to the elems and second depending on
+        fn
+    '''
+    return theano.map(fn, elems, name=name)[0]
+
+
+def foldl(fn, elems, initializer=None, name=None):
+    '''Reduce elems using fn to combine them from left to right.
+
+    # Arguments
+        fn: Callable that will be called upon each element in elems and an
+            accumulator, for instance lambda acc, x: acc + x
+        elems: tensor
+        initializer: The first value used (elems[0] in case of None)
+        name: A string name for the foldl node in the graph
+
+    # Returns
+        Same type and shape as initializer
+    '''
+    if initializer is None:
+        initializer = elems[0]
+        elems = elems[1:]
+
+    # We need to change the order of the arguments because theano accepts x as
+    # first parameter and accumulator as second
+    fn2 = lambda x, acc: fn(acc, x)
+
+    return theano.foldl(fn2, elems, initializer, name=name)[0]
+
+
+def foldr(fn, elems, initializer=None, name=None):
+    '''Reduce elems using fn to combine them from right to left.
+
+    # Arguments
+        fn: Callable that will be called upon each element in elems and an
+            accumulator, for instance lambda acc, x: acc + x
+        elems: tensor
+        initializer: The first value used (elems[-1] in case of None)
+        name: A string name for the foldr node in the graph
+
+    # Returns
+        Same type and shape as initializer
+    '''
+    if initializer is None:
+        initializer = elems[-1]
+        elems = elems[:-1]
+
+    # We need to change the order of the arguments because theano accepts x as
+    # first parameter and accumulator as second
+    fn2 = lambda x, acc: fn(acc, x)
+
+    return theano.foldr(fn2, elems, initializer, name=name)[0]

keras/callbacks.py

@@ -1,12 +1,15 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+import os
+import csv
+
 import numpy as np
 import time
 import json
 import warnings
 
-from collections import deque
+from collections import deque, OrderedDict, Iterable
 from .utils.generic_utils import Progbar
 from keras import backend as K
 from pkg_resources import parse_version
@@ -312,6 +315,10 @@ class EarlyStopping(Callback):
 
     # Arguments
         monitor: quantity to be monitored.
+        min_delta: minimum change in the monitored quantity
+            to qualify as an improvement, i.e. an absolute
+            change of less than min_delta, will count as no
+            improvement.
         patience: number of epochs with no improvement
             after which training will be stopped.
         verbose: verbosity mode.
@@ -323,13 +330,15 @@ class EarlyStopping(Callback):
             mode, the direction is automatically inferred
             from the name of the monitored quantity.
     '''
-    def __init__(self, monitor='val_loss', patience=0, verbose=0, mode='auto'):
+    def __init__(self, monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto'):
         super(EarlyStopping, self).__init__()
 
         self.monitor = monitor
         self.patience = patience
         self.verbose = verbose
+        self.min_delta = min_delta
         self.wait = 0
+        self.stopped_epoch = 0
 
         if mode not in ['auto', 'min', 'max']:
             warnings.warn('EarlyStopping mode %s is unknown, '
@@ -347,6 +356,11 @@ class EarlyStopping(Callback):
             else:
                 self.monitor_op = np.less
 
+        if self.monitor_op == np.greater:
+            self.min_delta *= 1
+        else:
+            self.min_delta *= -1
+
     def on_train_begin(self, logs={}):
         self.wait = 0       # Allow instances to be re-used
         self.best = np.Inf if self.monitor_op == np.less else -np.Inf
@@ -357,16 +371,19 @@ class EarlyStopping(Callback):
             warnings.warn('Early stopping requires %s available!' %
                           (self.monitor), RuntimeWarning)
 
-        if self.monitor_op(current, self.best):
+        if self.monitor_op(current - self.min_delta, self.best):
             self.best = current
             self.wait = 0
         else:
             if self.wait >= self.patience:
-                if self.verbose > 0:
-                    print('Epoch %05d: early stopping' % (epoch))
+                self.stopped_epoch = epoch
                 self.model.stop_training = True
             self.wait += 1
 
+    def on_train_end(self, logs={}):
+        if self.stopped_epoch > 0 and self.verbose > 0:
+            print('Epoch %05d: early stopping' % (self.stopped_epoch))
+
 
 class RemoteMonitor(Callback):
     '''Callback used to stream events to a server.
@@ -420,7 +437,11 @@ class LearningRateScheduler(Callback):
         assert hasattr(self.model.optimizer, 'lr'), \
             'Optimizer must have a "lr" attribute.'
         lr = self.schedule(epoch)
-        assert type(lr) == float, 'The output of the "schedule" function should be float.'
+
+        if not isinstance(lr, (float, np.float32, np.float64)):
+            raise ValueError('The output of the "schedule" function '
+                             'should be float.')
+
         K.set_value(self.model.optimizer.lr, lr)
 
 
@@ -528,7 +549,226 @@ class TensorBoard(Callback):
                 continue
             summary = tf.Summary()
             summary_value = summary.value.add()
-            summary_value.simple_value = value
+            summary_value.simple_value = value.item()
             summary_value.tag = name
             self.writer.add_summary(summary, epoch)
         self.writer.flush()
+
+
+class ReduceLROnPlateau(Callback):
+    '''Reduce learning rate when a metric has stopped improving.
+
+    Models often benefit from reducing the learning rate by a factor
+    of 2-10 once learning stagnates. This callback monitors a
+    quantity and if no improvement is seen for a 'patience' number
+    of epochs, the learning rate is reduced.
+
+    # Example
+        ```python
+            reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
+                                          patience=5, min_lr=0.001)
+            model.fit(X_train, Y_train, callbacks=[reduce_lr])
+        ```
+
+    # Arguments
+        monitor: quantity to be monitored.
+        factor: factor by which the learning rate will
+            be reduced. new_lr = lr * factor
+        patience: number of epochs with no improvement
+            after which learning rate will be reduced.
+        verbose: int. 0: quiet, 1: update messages.
+        mode: one of {auto, min, max}. In `min` mode,
+            lr will be reduced when the quantity
+            monitored has stopped decreasing; in `max`
+            mode it will be reduced when the quantity
+            monitored has stopped increasing; in `auto`
+            mode, the direction is automatically inferred
+            from the name of the monitored quantity.
+        epsilon: threshold for measuring the new optimum,
+            to only focus on significant changes.
+        cooldown: number of epochs to wait before resuming
+            normal operation after lr has been reduced.
+        min_lr: lower bound on the learning rate.
+    '''
+
+    def __init__(self, monitor='val_loss', factor=0.1, patience=10,
+                 verbose=0, mode='auto', epsilon=1e-4, cooldown=0, min_lr=0):
+        super(Callback, self).__init__()
+
+        self.monitor = monitor
+        if factor >= 1.0:
+            raise ValueError('ReduceLROnPlateau does not support a factor >= 1.0.')
+        self.factor = factor
+        self.min_lr = min_lr
+        self.epsilon = epsilon
+        self.patience = patience
+        self.verbose = verbose
+        self.cooldown = cooldown
+        self.cooldown_counter = 0  # Cooldown counter.
+        self.wait = 0
+        self.best = 0
+        self.mode = mode
+        self.monitor_op = None
+        self.reset()
+
+    def reset(self):
+        if self.mode not in ['auto', 'min', 'max']:
+            warnings.warn('Learning Rate Plateau Reducing mode %s is unknown, '
+                          'fallback to auto mode.' % (self.mode), RuntimeWarning)
+            self.mode = 'auto'
+        if self.mode == 'min' or (self.mode == 'auto' and 'acc' not in self.monitor):
+            self.monitor_op = lambda a, b: np.less(a, b - self.epsilon)
+            self.best = np.Inf
+        else:
+            self.monitor_op = lambda a, b: np.greater(a, b + self.epsilon)
+            self.best = -np.Inf
+        self.cooldown_counter = 0
+        self.wait = 0
+        self.lr_epsilon = self.min_lr * 1e-4
+
+    def on_train_begin(self, logs={}):
+        self.reset()
+
+    def on_epoch_end(self, epoch, logs={}):
+        logs['lr'] = K.get_value(self.model.optimizer.lr)
+        current = logs.get(self.monitor)
+        if current is None:
+            warnings.warn('Learning Rate Plateau Reducing requires %s available!' %
+                          self.monitor, RuntimeWarning)
+        else:
+            if self.in_cooldown():
+                self.cooldown_counter -= 1
+                self.wait = 0
+
+            if self.monitor_op(current, self.best):
+                self.best = current
+                self.wait = 0
+            elif not self.in_cooldown():
+                if self.wait >= self.patience:
+                    old_lr = float(K.get_value(self.model.optimizer.lr))
+                    if old_lr > self.min_lr + self.lr_epsilon:
+                        new_lr = old_lr * self.factor
+                        new_lr = max(new_lr, self.min_lr)
+                        K.set_value(self.model.optimizer.lr, new_lr)
+                        if self.verbose > 0:
+                            print('\nEpoch %05d: reducing learning rate to %s.' % (epoch, new_lr))
+                        self.cooldown_counter = self.cooldown
+                        self.wait = 0
+                self.wait += 1
+
+    def in_cooldown(self):
+        return self.cooldown_counter > 0
+
+
+class CSVLogger(Callback):
+    '''Callback that streams epoch results to a csv file.
+    Supports all values that can be represented as a string,
+    including 1D iterables such as np.ndarray.
+
+    # Example
+        ```python
+            csv_logger = CSVLogger('training.log')
+            model.fit(X_train, Y_train, callbacks=[csv_logger])
+        ```
+
+    Arguments
+        filename: filename of the csv file, e.g. 'run/log.csv'.
+        separator: string used to separate elements in the csv file.
+        append: True: append if file exists (useful for continuing
+            training). False: overwrite existing file,
+    '''
+
+    def __init__(self, filename, separator=',', append=False):
+        self.sep = separator
+        self.filename = filename
+        self.append = append
+        self.writer = None
+        self.keys = None
+        self.append_header = True
+        super(CSVLogger, self).__init__()
+
+    def on_train_begin(self, logs={}):
+        if self.append:
+            if os.path.exists(self.filename):
+                with open(self.filename) as f:
+                    self.append_header = len(f.readline()) == 0
+            self.csv_file = open(self.filename, 'a')
+        else:
+            self.csv_file = open(self.filename, 'w')
+
+    def on_epoch_end(self, epoch, logs={}):
+        def handle_value(k):
+            is_zero_dim_ndarray = isinstance(k, np.ndarray) and k.ndim == 0
+            if isinstance(k, Iterable) and not is_zero_dim_ndarray:
+                return '"[%s]"' % (', '.join(map(lambda x: str(x), k)))
+            else:
+                return k
+
+        if not self.writer:
+            self.keys = sorted(logs.keys())
+            self.writer = csv.DictWriter(self.csv_file, fieldnames=['epoch'] + self.keys)
+            if self.append_header:
+                self.writer.writeheader()
+
+        row_dict = OrderedDict({'epoch': epoch})
+        row_dict.update((key, handle_value(logs[key])) for key in self.keys)
+        self.writer.writerow(row_dict)
+        self.csv_file.flush()
+
+    def on_train_end(self, logs={}):
+        self.csv_file.close()
+
+
+class LambdaCallback(Callback):
+    """Callback for creating simple, custom callbacks on-the-fly.
+
+    This callback is constructed with anonymous functions that will be called
+    at the appropiate time. Note that the callbacks expects positional
+    arguments, as:
+     - `on_epoch_begin` and `on_epoch_end` expect two positional arguments: `epoch`, `logs`
+     - `on_batch_begin` and `on_batch_end` expect two positional arguments: `batch`, `logs`
+     - `on_train_begin` and `on_train_end` expect one positional argument: `logs`
+
+    # Arguments
+        on_epoch_begin: called at the beginning of every epoch.
+        on_epoch_end: called at the end of every epoch.
+        on_batch_begin: called at the beginning of every batch.
+        on_batch_end: called at the end of every batch.
+        on_train_begin: called at the beginning of model training.
+        on_train_end: called at the end of model training.
+
+    # Example
+        ```python
+        # Print the batch number at the beginning of every batch.
+        batch_print_callback = LambdaCallback(on_batch_begin=lambda batch, logs: print(batch))
+
+        # Plot the loss after every epoch.
+        import numpy as np
+        import matplotlib.pyplot as plt
+        plot_loss_callback = LambdaCallback(on_epoch_end=lambda epoch, logs: plt.plot(np.arange(epoch), logs['loss']))
+
+        # Terminate some processes after having finished model training.
+        processes = ...
+        cleanup_callback = LambdaCallback(on_train_end=lambda logs: [p.terminate() for p in processes if p.is_alive()])
+
+        model.fit(..., callbacks=[batch_print_callback, plot_loss_callback, cleanup_callback])
+        ```
+
+    """
+
+    def __init__(self,
+                 on_epoch_begin=None,
+                 on_epoch_end=None,
+                 on_batch_begin=None,
+                 on_batch_end=None,
+                 on_train_begin=None,
+                 on_train_end=None,
+                 **kwargs):
+        super(Callback, self).__init__()
+        self.__dict__.update(kwargs)
+        self.on_epoch_begin = on_epoch_begin if on_epoch_begin else lambda epoch, logs: None
+        self.on_epoch_end = on_epoch_end if on_epoch_end else lambda epoch, logs: None
+        self.on_batch_begin = on_batch_begin if on_batch_begin else lambda batch, logs: None
+        self.on_batch_end = on_batch_end if on_batch_end else lambda batch, logs: None
+        self.on_train_begin = on_train_begin if on_train_begin else lambda logs: None
+        self.on_train_end = on_train_end if on_train_end else lambda logs: None

keras/datasets/cifar.py

@@ -11,9 +11,10 @@ def load_batch(fpath, label_key='labels'):
     else:
         d = cPickle.load(f, encoding="bytes")
         # decode utf8
+        d_decoded = {}
         for k, v in d.items():
-            del(d[k])
-            d[k.decode("utf8")] = v
+            d_decoded[k.decode("utf8")] = v
+        d = d_decoded
     f.close()
     data = d["data"]
     labels = d[label_key]

keras/datasets/mnist.py

@@ -1,14 +1,13 @@
-# -*- coding: utf-8 -*-
 import gzip
 from ..utils.data_utils import get_file
 from six.moves import cPickle
 import sys
 
 
-def load_data(path="mnist.pkl.gz"):
-    path = get_file(path, origin="https://s3.amazonaws.com/img-datasets/mnist.pkl.gz")
+def load_data(path='mnist.pkl.gz'):
+    path = get_file(path, origin='https://s3.amazonaws.com/img-datasets/mnist.pkl.gz')
 
-    if path.endswith(".gz"):
+    if path.endswith('.gz'):
         f = gzip.open(path, 'rb')
     else:
         f = open(path, 'rb')
@@ -16,7 +15,7 @@ def load_data(path="mnist.pkl.gz"):
     if sys.version_info < (3,):
         data = cPickle.load(f)
     else:
-        data = cPickle.load(f, encoding="bytes")
+        data = cPickle.load(f, encoding='bytes')
 
     f.close()
     return data  # (X_train, y_train), (X_test, y_test)

keras/datasets/reuters.py

@@ -10,7 +10,8 @@ import sys
 def load_data(path='reuters.pkl', nb_words=None, skip_top=0,
               maxlen=None, test_split=0.2, seed=113,
               start_char=1, oov_char=2, index_from=3):
-    '''
+    '''Loads the Reuters newswire classification dataset.
+
     # Arguments
         path: where to store the data (in `/.keras/dataset`)
         nb_words: max number of words to include. Words are ranked

keras/engine/training.py

@@ -7,6 +7,9 @@ import time
 import numpy as np
 import multiprocessing
 import threading
+
+import six
+
 try:
     import queue
 except ImportError:
@@ -183,13 +186,12 @@ def check_array_lengths(X, Y, W):
 
 
 def check_loss_and_target_compatibility(targets, losses, output_shapes):
-    assert len(targets) == len(losses) == len(output_shapes)
     key_losses = {'mean_square_error',
                   'binary_crossentropy',
                   'categorical_crossentropy'}
     for y, loss, shape in zip(targets, losses, output_shapes):
         if loss.__name__ == 'categorical_crossentropy':
-            if y.shape[1] == 1:
+            if y.shape[-1] == 1:
                 raise Exception('You are passing a target array of shape ' + str(y.shape) +
                                 ' while using as loss `categorical_crossentropy`. '
                                 '`categorical_crossentropy` expects '
@@ -205,13 +207,15 @@ def check_loss_and_target_compatibility(targets, losses, output_shapes):
                                 'Alternatively, you can use the loss function '
                                 '`sparse_categorical_crossentropy` instead, '
                                 'which does expect integer targets.')
-        if loss.__name__ in key_losses and shape[1] is not None and y.shape[1] != shape[1]:
-            raise Exception('A target array with shape ' + str(y.shape) +
-                            ' was passed for an output of shape ' + str(shape) +
-                            ' while using as loss `' + loss.__name__ + '`. '
-                            'This loss expects '
-                            'targets to have the same shape '
-                            'as the output.')
+        if loss.__name__ in key_losses:
+            for target_dim, out_dim in zip(y.shape[1:], shape[1:]):
+                if out_dim is not None and target_dim != out_dim:
+                    raise Exception('A target array with shape ' + str(y.shape) +
+                                    ' was passed for an output of shape ' + str(shape) +
+                                    ' while using as loss `' + loss.__name__ + '`. '
+                                    'This loss expects '
+                                    'targets to have the same shape '
+                                    'as the output.')
 
 
 def collect_metrics(metrics, output_names):
@@ -234,31 +238,6 @@ def collect_metrics(metrics, output_names):
                         str(metrics))
 
 
-def collect_trainable_weights(layer):
-    '''Collects all `trainable_weights` attributes,
-    excluding any sublayers where `trainable` is set the `False`.
-    '''
-    trainable = getattr(layer, 'trainable', True)
-    if not trainable:
-        return []
-    weights = []
-    if layer.__class__.__name__ == 'Sequential':
-        for sublayer in layer.flattened_layers:
-            weights += collect_trainable_weights(sublayer)
-    elif layer.__class__.__name__ == 'Model':
-        for sublayer in layer.layers:
-            weights += collect_trainable_weights(sublayer)
-    elif layer.__class__.__name__ == 'Graph':
-        for sublayer in layer._graph_nodes.values():
-            weights += collect_trainable_weights(sublayer)
-    else:
-        weights += layer.trainable_weights
-    # dedupe weights
-    weights = list(set(weights))
-    weights.sort(key=lambda x: x.name)
-    return weights
-
-
 def batch_shuffle(index_array, batch_size):
     '''This shuffles an array in a batch-wise fashion.
     Useful for shuffling HDF5 arrays
@@ -450,7 +429,7 @@ def generator_queue(generator, max_q_size=10,
             q.close()
         raise
 
-    return q, _stop
+    return q, _stop, generator_threads
 
 
 class Model(Container):
@@ -602,7 +581,10 @@ class Model(Container):
         for i in range(len(self.outputs)):
             shape = self.internal_output_shapes[i]
             name = self.output_names[i]
-            self.targets.append(K.placeholder(ndim=len(shape), name=name + '_target'))
+            self.targets.append(K.placeholder(ndim=len(shape),
+                                name=name + '_target',
+                                sparse=K.is_sparse(self.outputs[i]),
+                                dtype=K.dtype(self.outputs[i])))
 
         # prepare metrics
         self.metrics = metrics
@@ -635,6 +617,15 @@ class Model(Container):
         # list of same size as output_names.
         # contains tuples (metrics for output, names of metrics)
         nested_metrics = collect_metrics(metrics, self.output_names)
+
+        def append_metric(layer_num, metric_name, metric_tensor):
+            """Helper function, used in loop below"""
+            if len(self.output_names) > 1:
+                metric_name = self.output_layers[layer_num].name + '_' + metric_name
+
+            self.metrics_names.append(metric_name)
+            self.metrics_tensors.append(metric_tensor)
+
         for i in range(len(self.outputs)):
             y_true = self.targets[i]
             y_pred = self.outputs[i]
@@ -644,27 +635,28 @@ class Model(Container):
                 if metric == 'accuracy' or metric == 'acc':
                     # custom handling of accuracy (because of class mode duality)
                     output_shape = self.internal_output_shapes[i]
+                    acc_fn = None
                     if output_shape[-1] == 1 or self.loss_functions[i] == objectives.binary_crossentropy:
                         # case: binary accuracy
-                        self.metrics_tensors.append(metrics_module.binary_accuracy(y_true, y_pred))
+                        acc_fn = metrics_module.binary_accuracy
                     elif self.loss_functions[i] == objectives.sparse_categorical_crossentropy:
                         # case: categorical accuracy with sparse targets
-                        self.metrics_tensors.append(
-                            metrics_module.sparse_categorical_accuracy(y_true, y_pred))
+                        acc_fn = metrics_module.sparse_categorical_accuracy
                     else:
-                        # case: categorical accuracy with dense targets
-                        self.metrics_tensors.append(metrics_module.categorical_accuracy(y_true, y_pred))
-                    if len(self.output_names) == 1:
-                        self.metrics_names.append('acc')
-                    else:
-                        self.metrics_names.append(self.output_layers[i].name + '_acc')
+                        acc_fn = metrics_module.categorical_accuracy
+
+                    append_metric(i, 'acc', acc_fn(y_true, y_pred))
                 else:
                     metric_fn = metrics_module.get(metric)
-                    self.metrics_tensors.append(metric_fn(y_true, y_pred))
-                    if len(self.output_names) == 1:
-                        self.metrics_names.append(metric_fn.__name__)
-                    else:
-                        self.metrics_names.append(self.output_layers[i].name + '_' + metric_fn.__name__)
+                    metric_result = metric_fn(y_true, y_pred)
+
+                    if not isinstance(metric_result, dict):
+                        metric_result = {
+                            metric_fn.__name__: metric_result
+                        }
+
+                    for name, tensor in six.iteritems(metric_result):
+                        append_metric(i, name, tensor)
 
         # prepare gradient updates and state updates
         self.optimizer = optimizers.get(optimizer)
@@ -680,7 +672,15 @@ class Model(Container):
         self.test_function = None
         self.predict_function = None
 
-        self._collected_trainable_weights = collect_trainable_weights(self)
+        # collected trainable weights and sort them deterministically.
+        trainable_weights = self.trainable_weights
+        # Sort weights by name
+        if trainable_weights:
+            if K.backend() == 'theano':
+                trainable_weights.sort(key=lambda x: x.name if x.name else x.auto_name)
+            else:
+                trainable_weights.sort(key=lambda x: x.name)
+        self._collected_trainable_weights = trainable_weights
 
     def _make_train_function(self):
         if not hasattr(self, 'train_function'):
@@ -736,7 +736,7 @@ class Model(Container):
     def _fit_loop(self, f, ins, out_labels=[], batch_size=32,
                   nb_epoch=100, verbose=1, callbacks=[],
                   val_f=None, val_ins=None, shuffle=True,
-                  callback_metrics=[]):
+                  callback_metrics=[], initial_epoch=0):
         '''Abstract fit function for f(ins).
         Assume that f returns a list, labeled by out_labels.
 
@@ -756,6 +756,8 @@ class Model(Container):
                 passed to the callbacks. They should be the
                 concatenation of list the display names of the outputs of
                  `f` and the list of display names of the outputs of `f_val`.
+            initial_epoch: epoch at which to start training
+                (useful for resuming a previous training run)
 
         # Returns
             `History` object.
@@ -796,7 +798,7 @@ class Model(Container):
         callback_model.stop_training = False
         self.validation_data = val_ins
 
-        for epoch in range(nb_epoch):
+        for epoch in range(initial_epoch, nb_epoch):
             callbacks.on_epoch_begin(epoch)
             if shuffle == 'batch':
                 index_array = batch_shuffle(index_array, batch_size)
@@ -983,7 +985,7 @@ class Model(Container):
 
     def fit(self, x, y, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
             validation_split=0., validation_data=None, shuffle=True,
-            class_weight=None, sample_weight=None):
+            class_weight=None, sample_weight=None, initial_epoch=0):
         '''Trains the model for a fixed number of epochs (iterations on a dataset).
 
         # Arguments
@@ -1007,7 +1009,7 @@ class Model(Container):
                 on this data at the end of each epoch.
             validation_data: data on which to evaluate the loss and any model metrics
                 at the end of each epoch. The model will not be trained on this data.
-                This could be a tuple (x_val, y_val) or a tuple (val_x, val_y, val_sample_weights).
+                This could be a tuple (x_val, y_val) or a tuple (x_val, y_val, val_sample_weights).
             shuffle: boolean, whether to shuffle the training data before each epoch.
             class_weight: optional dictionary mapping class indices (integers) to
                 a weight (float) to apply to the model's loss for the samples
@@ -1020,6 +1022,8 @@ class Model(Container):
                 with shape (samples, sequence_length),
                 to apply a different weight to every timestep of every sample.
                 In this case you should make sure to specify sample_weight_mode="temporal" in compile().
+            initial_epoch: epoch at which to start training
+                (useful for resuming a previous training run)
 
 
         # Returns
@@ -1103,7 +1107,8 @@ class Model(Container):
                               batch_size=batch_size, nb_epoch=nb_epoch,
                               verbose=verbose, callbacks=callbacks,
                               val_f=val_f, val_ins=val_ins, shuffle=shuffle,
-                              callback_metrics=callback_metrics)
+                              callback_metrics=callback_metrics,
+                              initial_epoch=initial_epoch)
 
     def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):
         '''Returns the loss value and metrics values for the model
@@ -1279,7 +1284,8 @@ class Model(Container):
     def fit_generator(self, generator, samples_per_epoch, nb_epoch,
                       verbose=1, callbacks=[],
                       validation_data=None, nb_val_samples=None,
-                      class_weight={}, max_q_size=10, nb_worker=1, pickle_safe=False):
+                      class_weight={}, max_q_size=10, nb_worker=1, pickle_safe=False,
+                      initial_epoch=0):
         '''Fits the model on data generated batch-by-batch by
         a Python generator.
         The generator is run in parallel to the model, for efficiency.
@@ -1315,6 +1321,8 @@ class Model(Container):
                 this implementation relies on multiprocessing, you should not pass
                 non picklable arguments to the generator as they can't be passed
                 easily to children processes.
+            initial_epoch: epoch at which to start training
+                (useful for resuming a previous training run)
 
         # Returns
             A `History` object.
@@ -1337,7 +1345,7 @@ class Model(Container):
         ```
         '''
         wait_time = 0.01  # in seconds
-        epoch = 0
+        epoch = initial_epoch
 
         do_validation = bool(validation_data)
         self._make_train_function()
@@ -1393,8 +1401,8 @@ class Model(Container):
             self.validation_data = None
 
         # start generator thread storing batches into a queue
-        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
-                                                pickle_safe=pickle_safe)
+        data_gen_queue, _stop, generator_threads = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
+                                                                   pickle_safe=pickle_safe)
 
         callback_model.stop_training = False
         while epoch < nb_epoch:
@@ -1468,7 +1476,9 @@ class Model(Container):
                     if val_gen:
                         val_outs = self.evaluate_generator(validation_data,
                                                            nb_val_samples,
-                                                           max_q_size=max_q_size)
+                                                           max_q_size=max_q_size,
+                                                           nb_worker=nb_worker,
+                                                           pickle_safe=pickle_safe)
                     else:
                         # no need for try/except because
                         # data has already been validated
@@ -1489,6 +1499,10 @@ class Model(Container):
 
         _stop.set()
         if pickle_safe:
+            # Terminate all daemon processes
+            for p in generator_threads:
+                if p.is_alive():
+                    p.terminate()
             data_gen_queue.close()
         callbacks.on_train_end()
         return self.history
@@ -1523,8 +1537,8 @@ class Model(Container):
         wait_time = 0.01
         all_outs = []
         weights = []
-        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
-                                                pickle_safe=pickle_safe)
+        data_gen_queue, _stop, generator_threads = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
+                                                                   pickle_safe=pickle_safe)
 
         while processed_samples < val_samples:
             generator_output = None
@@ -1569,6 +1583,10 @@ class Model(Container):
 
         _stop.set()
         if pickle_safe:
+            # Terminate all daemon processes
+            for p in generator_threads:
+                if p.is_alive():
+                    p.terminate()
             data_gen_queue.close()
         if type(outs) is not list:
             return np.average(np.asarray(all_outs),
@@ -1604,8 +1622,8 @@ class Model(Container):
         processed_samples = 0
         wait_time = 0.01
         all_outs = []
-        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
-                                                pickle_safe=pickle_safe)
+        data_gen_queue, _stop, generator_threads = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
+                                                                   pickle_safe=pickle_safe)
 
         while processed_samples < val_samples:
             generator_output = None
@@ -1658,6 +1676,10 @@ class Model(Container):
 
         _stop.set()
         if pickle_safe:
+            # Terminate all daemon processes
+            for p in generator_threads:
+                if p.is_alive():
+                    p.terminate()
             data_gen_queue.close()
         if len(all_outs) == 1:
             return all_outs[0]

keras/initializations.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import numpy as np
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def get_fans(shape, dim_ordering='th'):
@@ -20,7 +21,7 @@ def get_fans(shape, dim_ordering='th'):
             fan_in = shape[-2] * receptive_field_size
             fan_out = shape[-1] * receptive_field_size
         else:
-            raise Exception('Invalid dim_ordering: ' + dim_ordering)
+            raise ValueError('Invalid dim_ordering: ' + dim_ordering)
     else:
         # no specific assumptions
         fan_in = np.sqrt(np.prod(shape))
@@ -101,7 +102,6 @@ def one(shape, name=None):
     return K.ones(shape, name=name)
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier, **kwargs):
     return get_from_module(identifier, globals(),
                            'initialization', kwargs=kwargs)

keras/layers/__init__.py

@@ -10,3 +10,4 @@ from .embeddings import *
 from .noise import *
 from .advanced_activations import *
 from .wrappers import *
+from .convolutional_recurrent import *

keras/layers/advanced_activations.py

@@ -107,9 +107,7 @@ class ELU(Layer):
         super(ELU, self).__init__(**kwargs)
 
     def call(self, x, mask=None):
-        pos = K.relu(x)
-        neg = (x - abs(x)) * 0.5
-        return pos + self.alpha * (K.exp(neg) - 1.)
+        return K.elu(x, self.alpha)
 
     def get_config(self):
         config = {'alpha': float(self.alpha)}

keras/layers/convolutional.py

@@ -47,7 +47,7 @@ class Convolution1D(Layer):
             If you don't specify anything, no activation is applied
             (ie. "linear" activation: a(x) = x).
         weights: list of numpy arrays to set as initial weights.
-        border_mode: 'valid' or 'same'.
+        border_mode: 'valid', 'same' or 'full'. ('full' requires the Theano backend.)
         subsample_length: factor by which to subsample output.
         W_regularizer: instance of [WeightRegularizer](../regularizers.md)
             (eg. L1 or L2 regularization), applied to the main weights matrix.
@@ -77,19 +77,18 @@ class Convolution1D(Layer):
         `steps` value might have changed due to padding.
     '''
     def __init__(self, nb_filter, filter_length,
-                 init='uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample_length=1,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, input_dim=None, input_length=None, **kwargs):
 
-        if border_mode not in {'valid', 'same'}:
+        if border_mode not in {'valid', 'same', 'full'}:
             raise Exception('Invalid border mode for Convolution1D:', border_mode)
         self.nb_filter = nb_filter
         self.filter_length = filter_length
         self.init = initializations.get(init, dim_ordering='th')
         self.activation = activations.get(activation)
-        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
         self.border_mode = border_mode
         self.subsample_length = subsample_length
 
@@ -143,6 +142,7 @@ class Convolution1D(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         length = conv_output_length(input_shape[1],
@@ -218,7 +218,7 @@ class AtrousConvolution1D(Convolution1D):
             If you don't specify anything, no activation is applied
             (ie. "linear" activation: a(x) = x).
         weights: list of numpy arrays to set as initial weights.
-        border_mode: 'valid' or 'same'.
+        border_mode: 'valid', 'same' or 'full'. ('full' requires the Theano backend.)
         subsample_length: factor by which to subsample output.
         atrous_rate: Factor for kernel dilation. Also called filter_dilation
             elsewhere.
@@ -250,13 +250,13 @@ class AtrousConvolution1D(Convolution1D):
         `steps` value might have changed due to padding.
     '''
     def __init__(self, nb_filter, filter_length,
-                 init='uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample_length=1, atrous_rate=1,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, **kwargs):
 
-        if border_mode not in {'valid', 'same'}:
+        if border_mode not in {'valid', 'same', 'full'}:
             raise Exception('Invalid border mode for AtrousConv1D:', border_mode)
 
         self.atrous_rate = int(atrous_rate)
@@ -331,7 +331,7 @@ class Convolution2D(Layer):
             If you don't specify anything, no activation is applied
             (ie. "linear" activation: a(x) = x).
         weights: list of numpy arrays to set as initial weights.
-        border_mode: 'valid' or 'same'.
+        border_mode: 'valid', 'same' or 'full'. ('full' requires the Theano backend.)
         subsample: tuple of length 2. Factor by which to subsample output.
             Also called strides elsewhere.
         W_regularizer: instance of [WeightRegularizer](../regularizers.md)
@@ -348,7 +348,7 @@ class Convolution2D(Layer):
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
         bias: whether to include a bias
             (i.e. make the layer affine rather than linear).
 
@@ -366,21 +366,20 @@ class Convolution2D(Layer):
         `rows` and `cols` values might have changed due to padding.
     '''
     def __init__(self, nb_filter, nb_row, nb_col,
-                 init='glorot_uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample=(1, 1), dim_ordering='default',
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, **kwargs):
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
-        if border_mode not in {'valid', 'same'}:
+        if border_mode not in {'valid', 'same', 'full'}:
             raise Exception('Invalid border mode for Convolution2D:', border_mode)
         self.nb_filter = nb_filter
         self.nb_row = nb_row
         self.nb_col = nb_col
         self.init = initializations.get(init, dim_ordering=dim_ordering)
         self.activation = activations.get(activation)
-        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
         self.border_mode = border_mode
         self.subsample = tuple(subsample)
         assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
@@ -436,6 +435,7 @@ class Convolution2D(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         if self.dim_ordering == 'th':
@@ -506,19 +506,39 @@ class Deconvolution2D(Convolution2D):
     (tuple of integers, does not include the sample axis),
     e.g. `input_shape=(3, 128, 128)` for 128x128 RGB pictures.
 
+    To pass the correct `output_shape` to this layer,
+    one could use a test model to predict and observe the actual output shape.
+
     # Examples
 
     ```python
         # apply a 3x3 transposed convolution with stride 1x1 and 3 output filters on a 12x12 image:
         model = Sequential()
         model.add(Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), border_mode='valid', input_shape=(3, 12, 12)))
-        # output_shape will be (None, 3, 14, 14)
+        # Note that you will have to change the output_shape depending on the backend used.
+
+        # we can predict with the model and print the shape of the array.
+        dummy_input = np.ones((32, 3, 12, 12))
+        # For TensorFlow dummy_input = np.ones((32, 12, 12, 3))
+        preds = model.predict(dummy_input)
+        print(preds.shape)
+        # Theano GPU: (None, 3, 13, 13)
+        # Theano CPU: (None, 3, 14, 14)
+        # TensorFlow: (None, 14, 14, 3)
 
         # apply a 3x3 transposed convolution with stride 2x2 and 3 output filters on a 12x12 image:
         model = Sequential()
         model.add(Deconvolution2D(3, 3, 3, output_shape=(None, 3, 25, 25), subsample=(2, 2), border_mode='valid', input_shape=(3, 12, 12)))
         model.summary()
-        # output_shape will be (None, 3, 25, 25)
+
+        # we can predict with the model and print the shape of the array.
+        dummy_input = np.ones((32, 3, 12, 12))
+        # For TensorFlow dummy_input = np.ones((32, 12, 12, 3))
+        preds = model.predict(dummy_input)
+        print(preds.shape)
+        # Theano GPU: (None, 3, 25, 25)
+        # Theano CPU: (None, 3, 25, 25)
+        # TensorFlow: (None, 25, 25, 3)
     ```
 
     # Arguments
@@ -536,6 +556,9 @@ class Deconvolution2D(Convolution2D):
                     p - padding size,
                     a - user-specified quantity used to distinguish between
                         the s different possible output sizes.
+             Because a is not specified explicitly and Theano and Tensorflow
+             use different values, it is better to use a dummy input and observe
+             the actual output shape of a layer as specified in the examples.
         init: name of initialization function for the weights of the layer
             (see [initializations](../initializations.md)), or alternatively,
             Theano function to use for weights initialization.
@@ -547,7 +570,7 @@ class Deconvolution2D(Convolution2D):
             If you don't specify anything, no activation is applied
             (ie. "linear" activation: a(x) = x).
         weights: list of numpy arrays to set as initial weights.
-        border_mode: 'valid' or 'same'.
+        border_mode: 'valid', 'same' or 'full'. ('full' requires the Theano backend.)
         subsample: tuple of length 2. Factor by which to oversample output.
             Also called strides elsewhere.
         W_regularizer: instance of [WeightRegularizer](../regularizers.md)
@@ -564,7 +587,7 @@ class Deconvolution2D(Convolution2D):
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
         bias: whether to include a bias (i.e. make the layer affine rather than linear).
 
     # Input shape
@@ -586,7 +609,7 @@ class Deconvolution2D(Convolution2D):
         [3] [Deconvolutional Networks](http://www.matthewzeiler.com/pubs/cvpr2010/cvpr2010.pdf)
     '''
     def __init__(self, nb_filter, nb_row, nb_col, output_shape,
-                 init='glorot_uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample=(1, 1),
                  dim_ordering='default',
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
@@ -594,7 +617,7 @@ class Deconvolution2D(Convolution2D):
                  bias=True, **kwargs):
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
-        if border_mode not in {'valid', 'same'}:
+        if border_mode not in {'valid', 'same', 'full'}:
             raise Exception('Invalid border mode for Deconvolution2D:', border_mode)
 
         self.output_shape_ = output_shape
@@ -610,19 +633,14 @@ class Deconvolution2D(Convolution2D):
 
     def get_output_shape_for(self, input_shape):
         if self.dim_ordering == 'th':
-            rows = input_shape[2]
-            cols = input_shape[3]
+            rows = self.output_shape_[2]
+            cols = self.output_shape_[3]
         elif self.dim_ordering == 'tf':
-            rows = input_shape[1]
-            cols = input_shape[2]
+            rows = self.output_shape_[1]
+            cols = self.output_shape_[2]
         else:
             raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
 
-        rows = conv_input_length(rows, self.nb_row,
-                                 self.border_mode, self.subsample[0])
-        cols = conv_input_length(cols, self.nb_col,
-                                 self.border_mode, self.subsample[1])
-
         if self.dim_ordering == 'th':
             return (input_shape[0], self.nb_filter, rows, cols)
         elif self.dim_ordering == 'tf':
@@ -647,7 +665,7 @@ class Deconvolution2D(Convolution2D):
         return output
 
     def get_config(self):
-        config = {'output_shape': self.output_shape}
+        config = {'output_shape': self.output_shape_}
         base_config = super(Deconvolution2D, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
@@ -685,7 +703,7 @@ class AtrousConvolution2D(Convolution2D):
             If you don't specify anything, no activation is applied
             (ie. "linear" activation: a(x) = x).
         weights: list of numpy arrays to set as initial weights.
-        border_mode: 'valid' or 'same'.
+        border_mode: 'valid', 'same' or 'full'. ('full' requires the Theano backend.)
         subsample: tuple of length 2. Factor by which to subsample output.
             Also called strides elsewhere.
         atrous_rate: tuple of length 2. Factor for kernel dilation.
@@ -704,7 +722,7 @@ class AtrousConvolution2D(Convolution2D):
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
         bias: whether to include a bias (i.e. make the layer affine rather than linear).
 
     # Input shape
@@ -724,7 +742,7 @@ class AtrousConvolution2D(Convolution2D):
         - [Multi-Scale Context Aggregation by Dilated Convolutions](https://arxiv.org/abs/1511.07122)
     '''
     def __init__(self, nb_filter, nb_row, nb_col,
-                 init='glorot_uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample=(1, 1),
                  atrous_rate=(1, 1), dim_ordering='default',
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
@@ -733,7 +751,7 @@ class AtrousConvolution2D(Convolution2D):
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
 
-        if border_mode not in {'valid', 'same'}:
+        if border_mode not in {'valid', 'same', 'full'}:
             raise Exception('Invalid border mode for AtrousConv2D:', border_mode)
 
         self.atrous_rate = tuple(atrous_rate)
@@ -853,7 +871,7 @@ class SeparableConvolution2D(Layer):
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
         bias: whether to include a bias
             (i.e. make the layer affine rather than linear).
 
@@ -871,7 +889,7 @@ class SeparableConvolution2D(Layer):
         `rows` and `cols` values might have changed due to padding.
     '''
     def __init__(self, nb_filter, nb_row, nb_col,
-                 init='glorot_uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample=(1, 1),
                  depth_multiplier=1, dim_ordering='default',
                  depthwise_regularizer=None, pointwise_regularizer=None,
@@ -966,6 +984,7 @@ class SeparableConvolution2D(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         if self.dim_ordering == 'th':
@@ -1050,7 +1069,7 @@ class Convolution3D(Layer):
             If you don't specify anything, no activation is applied
             (ie. "linear" activation: a(x) = x).
         weights: list of Numpy arrays to set as initial weights.
-        border_mode: 'valid' or 'same'.
+        border_mode: 'valid', 'same' or 'full'. ('full' requires the Theano backend.)
         subsample: tuple of length 3. Factor by which to subsample output.
             Also called strides elsewhere.
             Note: 'subsample' is implemented by slicing the output of conv3d with strides=(1,1,1).
@@ -1068,7 +1087,7 @@ class Convolution3D(Layer):
             (the depth) is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
         bias: whether to include a bias (i.e. make the layer affine rather than linear).
 
     # Input shape
@@ -1086,7 +1105,7 @@ class Convolution3D(Layer):
     '''
 
     def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3,
-                 init='glorot_uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample=(1, 1, 1), dim_ordering='default',
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
@@ -1094,7 +1113,7 @@ class Convolution3D(Layer):
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
 
-        if border_mode not in {'valid', 'same'}:
+        if border_mode not in {'valid', 'same', 'full'}:
             raise Exception('Invalid border mode for Convolution3D:', border_mode)
         self.nb_filter = nb_filter
         self.kernel_dim1 = kernel_dim1
@@ -1102,7 +1121,6 @@ class Convolution3D(Layer):
         self.kernel_dim3 = kernel_dim3
         self.init = initializations.get(init, dim_ordering=dim_ordering)
         self.activation = activations.get(activation)
-        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
         self.border_mode = border_mode
         self.subsample = tuple(subsample)
         assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
@@ -1164,6 +1182,7 @@ class Convolution3D(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         if self.dim_ordering == 'th':
@@ -1271,7 +1290,7 @@ class UpSampling2D(Layer):
             is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -1334,7 +1353,7 @@ class UpSampling3D(Layer):
             is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         5D tensor with shape:
@@ -1394,9 +1413,16 @@ class ZeroPadding1D(Layer):
     '''Zero-padding layer for 1D input (e.g. temporal sequence).
 
     # Arguments
-        padding: int
+        padding: int, or tuple of int (length 2), or dictionary.
+            - If int:
             How many zeros to add at the beginning and end of
             the padding dimension (axis 1).
+            - If tuple of int (length 2)
+            How many zeros to add at the beginning and at the end of
+            the padding dimension, in order '(left_pad, right_pad)'.
+            - If dictionary: should contain the keys
+            {'left_pad', 'right_pad'}.
+            If any key is missing, default value of 0 will be used for the missing key.
 
     # Input shape
         3D tensor with shape (samples, axis_to_pad, features)
@@ -1408,16 +1434,37 @@ class ZeroPadding1D(Layer):
     def __init__(self, padding=1, **kwargs):
         super(ZeroPadding1D, self).__init__(**kwargs)
         self.padding = padding
+
+        if isinstance(padding, int):
+            self.left_pad = padding
+            self.right_pad = padding
+
+        elif isinstance(padding, dict):
+            if set(padding.keys()) <= {'left_pad', 'right_pad'}:
+                self.left_pad = padding.get('left_pad', 0)
+                self.right_pad = padding.get('right_pad', 0)
+            else:
+                raise ValueError('Unexpected key found in `padding` dictionary. '
+                                 'Keys have to be in {"left_pad", "right_pad"}. '
+                                 'Found: ' + str(padding.keys()))
+        else:
+            padding = tuple(padding)
+            if len(padding) != 2:
+                raise ValueError('`padding` should be int, or dict with keys '
+                                 '{"left_pad", "right_pad"}, or tuple of length 2. '
+                                 'Found: ' + str(padding))
+            self.left_pad = padding[0]
+            self.right_pad = padding[1]
         self.input_spec = [InputSpec(ndim=3)]
 
     def get_output_shape_for(self, input_shape):
-        length = input_shape[1] + self.padding * 2 if input_shape[1] is not None else None
+        length = input_shape[1] + self.left_pad + self.right_pad if input_shape[1] is not None else None
         return (input_shape[0],
                 length,
                 input_shape[2])
 
     def call(self, x, mask=None):
-        return K.temporal_padding(x, padding=self.padding)
+        return K.asymmetric_temporal_padding(x, left_pad=self.left_pad, right_pad=self.right_pad)
 
     def get_config(self):
         config = {'padding': self.padding}
@@ -1429,55 +1476,103 @@ class ZeroPadding2D(Layer):
     '''Zero-padding layer for 2D input (e.g. picture).
 
     # Arguments
-        padding: tuple of int (length 2)
+        padding: tuple of int (length 2), or tuple of int (length 4), or dictionary.
+            - If tuple of int (length 2):
             How many zeros to add at the beginning and end of
-            the 2 padding dimensions (axis 3 and 4).
+            the 2 padding dimensions (rows and cols).
+            - If tuple of int (length 4):
+            How many zeros to add at the beginning and at the end of
+            the 2 padding dimensions (rows and cols), in the order
+            '(top_pad, bottom_pad, left_pad, right_pad)'.
+            - If dictionary: should contain the keys
+            {'top_pad', 'bottom_pad', 'left_pad', 'right_pad'}.
+            If any key is missing, default value of 0 will be used for the missing key.
         dim_ordering: 'th' or 'tf'.
             In 'th' mode, the channels dimension (the depth)
             is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
-        (samples, depth, first_axis_to_pad, second_axis_to_pad)
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
 
     # Output shape
         4D tensor with shape:
-        (samples, depth, first_padded_axis, second_padded_axis)
+        `(samples, channels, padded_rows, padded_cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, padded_rows, padded_cols, channels)` if dim_ordering='tf'.
     '''
 
-    def __init__(self, padding=(1, 1), dim_ordering='default', **kwargs):
+    def __init__(self,
+                 padding=(1, 1),
+                 dim_ordering='default',
+                 **kwargs):
         super(ZeroPadding2D, self).__init__(**kwargs)
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
-        self.padding = tuple(padding)
-        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+
+        self.padding = padding
+        if isinstance(padding, dict):
+            if set(padding.keys()) <= {'top_pad', 'bottom_pad', 'left_pad', 'right_pad'}:
+                self.top_pad = padding.get('top_pad', 0)
+                self.bottom_pad = padding.get('bottom_pad', 0)
+                self.left_pad = padding.get('left_pad', 0)
+                self.right_pad = padding.get('right_pad', 0)
+            else:
+                raise ValueError('Unexpected key found in `padding` dictionary. '
+                                 'Keys have to be in {"top_pad", "bottom_pad", '
+                                 '"left_pad", "right_pad"}.'
+                                 'Found: ' + str(padding.keys()))
+        else:
+            padding = tuple(padding)
+            if len(padding) == 2:
+                self.top_pad = padding[0]
+                self.bottom_pad = padding[0]
+                self.left_pad = padding[1]
+                self.right_pad = padding[1]
+            elif len(padding) == 4:
+                self.top_pad = padding[0]
+                self.bottom_pad = padding[1]
+                self.left_pad = padding[2]
+                self.right_pad = padding[3]
+            else:
+                raise TypeError('`padding` should be tuple of int '
+                                'of length 2 or 4, or dict. '
+                                'Found: ' + str(padding))
+
+        assert dim_ordering in {'tf', 'th'}, '`dim_ordering` must be in {"tf", "th"}.'
         self.dim_ordering = dim_ordering
         self.input_spec = [InputSpec(ndim=4)]
 
     def get_output_shape_for(self, input_shape):
         if self.dim_ordering == 'th':
-            width = input_shape[2] + 2 * self.padding[0] if input_shape[2] is not None else None
-            height = input_shape[3] + 2 * self.padding[1] if input_shape[3] is not None else None
+            rows = input_shape[2] + self.top_pad + self.bottom_pad if input_shape[2] is not None else None
+            cols = input_shape[3] + self.left_pad + self.right_pad if input_shape[3] is not None else None
             return (input_shape[0],
                     input_shape[1],
-                    width,
-                    height)
+                    rows,
+                    cols)
         elif self.dim_ordering == 'tf':
-            width = input_shape[1] + 2 * self.padding[0] if input_shape[1] is not None else None
-            height = input_shape[2] + 2 * self.padding[1] if input_shape[2] is not None else None
+            rows = input_shape[1] + self.top_pad + self.bottom_pad if input_shape[1] is not None else None
+            cols = input_shape[2] + self.left_pad + self.right_pad if input_shape[2] is not None else None
             return (input_shape[0],
-                    width,
-                    height,
+                    rows,
+                    cols,
                     input_shape[3])
         else:
             raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
 
     def call(self, x, mask=None):
-        return K.spatial_2d_padding(x, padding=self.padding,
-                                    dim_ordering=self.dim_ordering)
+        return K.asymmetric_spatial_2d_padding(x,
+                                               top_pad=self.top_pad,
+                                               bottom_pad=self.bottom_pad,
+                                               left_pad=self.left_pad,
+                                               right_pad=self.right_pad,
+                                               dim_ordering=self.dim_ordering)
 
     def get_config(self):
         config = {'padding': self.padding}
@@ -1492,12 +1587,13 @@ class ZeroPadding3D(Layer):
         padding: tuple of int (length 3)
             How many zeros to add at the beginning and end of
             the 3 padding dimensions (axis 3, 4 and 5).
+            Currentl only symmetric padding is supported.
         dim_ordering: 'th' or 'tf'.
             In 'th' mode, the channels dimension (the depth)
             is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         5D tensor with shape:
@@ -1572,6 +1668,7 @@ class Cropping1D(Layer):
 
     def build(self, input_shape):
         self.input_spec = [InputSpec(shape=input_shape)]
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         length = input_shape[1] - self.cropping[0] - self.cropping[1] if input_shape[1] is not None else None
@@ -1588,6 +1685,7 @@ class Cropping1D(Layer):
         base_config = super(Cropping1D, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
+
 class Cropping2D(Layer):
     '''Cropping layer for 2D input (e.g. picture).
     It crops along spatial dimensions, i.e. width and height.
@@ -1601,7 +1699,7 @@ class Cropping2D(Layer):
             is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -1640,6 +1738,7 @@ class Cropping2D(Layer):
 
     def build(self, input_shape):
         self.input_spec = [InputSpec(shape=input_shape)]
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         if self.dim_ordering == 'th':
@@ -1673,8 +1772,9 @@ class Cropping2D(Layer):
         base_config = super(Cropping2D, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
+
 class Cropping3D(Layer):
-    '''Cropping layer for 2D input (e.g. picture).
+    '''Cropping layer for 3D data (e.g. spatial or saptio-temporal).
 
     # Arguments
         cropping: tuple of tuple of int (length 3)
@@ -1685,7 +1785,7 @@ class Cropping3D(Layer):
             is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         5D tensor with shape:
@@ -1712,6 +1812,7 @@ class Cropping3D(Layer):
 
     def build(self, input_shape):
         self.input_spec = [InputSpec(shape=input_shape)]
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         if self.dim_ordering == 'th':

keras/layers/convolutional_recurrent.py

@@ -0,0 +1,516 @@
+from .. import backend as K
+from .. import activations, initializations, regularizers
+
+import numpy as np
+from ..engine import Layer, InputSpec
+from ..utils.np_utils import conv_output_length
+import warnings
+
+
+class ConvRecurrent2D(Layer):
+    '''Abstract base class for convolutional recurrent layers.
+    Do not use in a model -- it's not a functional layer!
+
+    ConvLSTM2D
+    follow the specifications of this class and accept
+    the keyword arguments listed below.
+
+    # Input shape
+        5D tensor with shape `(nb_samples, timesteps, channels, rows, cols)`.
+
+    # Output shape
+        - if `return_sequences`: 5D tensor with shape
+            `(nb_samples, timesteps, channels, rows, cols)`.
+        - else, 4D tensor with shape `(nb_samples, channels, rows, cols)`.
+
+    # Arguments
+        weights: list of numpy arrays to set as initial weights.
+            The list should have 3 elements, of shapes:
+            `[(input_dim, nb_filter), (nb_filter, nb_filter), (nb_filter,)]`.
+        return_sequences: Boolean. Whether to return the last output
+            in the output sequence, or the full sequence.
+        go_backwards: Boolean (default False).
+            If True, rocess the input sequence backwards.
+        stateful: Boolean (default False). If True, the last state
+            for each sample at index i in a batch will be used as initial
+            state for the sample of index i in the following batch.
+        nb_filter: Number of convolution filters to use.
+        nb_row: Number of rows in the convolution kernel.
+        nb_col: Number of columns in the convolution kernel.
+            is required when using this layer as the first layer in a model.
+        input_shape: input_shape
+
+    # Masking
+        This layer supports masking for input data with a variable number
+        of timesteps. To introduce masks to your data,
+        use an [Embedding](embeddings.md) layer with the `mask_zero` parameter
+        set to `True`.
+        **Note:** for the time being, masking is only supported with Theano.
+
+    # TensorFlow warning
+        For the time being, when using the TensorFlow backend,
+        the number of timesteps used must be specified in your model.
+        Make sure to pass an `input_length` int argument to your
+        recurrent layer (if it comes first in your model),
+        or to pass a complete `input_shape` argument to the first layer
+        in your model otherwise.
+
+
+    # Note on using statefulness in RNNs
+        You can set RNN layers to be 'stateful', which means that the states
+        computed for the samples in one batch will be reused as initial states
+        for the samples in the next batch.
+        This assumes a one-to-one mapping between
+        samples in different successive batches.
+
+        To enable statefulness:
+            - specify `stateful=True` in the layer constructor.
+            - specify a fixed batch size for your model, by passing
+                a `batch_input_size=(...)` to the first layer in your model.
+                This is the expected shape of your inputs *including the batch
+                size*.
+                It should be a tuple of integers, e.g. `(32, 10, 100)`.
+
+        To reset the states of your model, call `.reset_states()` on either
+        a specific layer, or on your entire model.
+    '''
+
+    def __init__(self, weights=None, nb_row=None, nb_col=None, nb_filter=None,
+                 return_sequences=False, go_backwards=False, stateful=False,
+                 dim_ordering=None, **kwargs):
+        self.return_sequences = return_sequences
+        self.go_backwards = go_backwards
+        self.stateful = stateful
+        self.initial_weights = weights
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.nb_filter = nb_filter
+        self.dim_ordering = dim_ordering
+        self.input_spec = [InputSpec(ndim=5)]
+
+        super(ConvRecurrent2D, self).__init__(**kwargs)
+
+    def compute_mask(self, input, mask):
+        if self.return_sequences:
+            return mask
+        else:
+            return None
+
+    def get_output_shape_for(self, input_shape):
+
+        if self.dim_ordering == 'th':
+            rows = input_shape[3]
+            cols = input_shape[4]
+        elif self.dim_ordering == 'tf':
+            rows = input_shape[2]
+            cols = input_shape[3]
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        rows = conv_output_length(rows, self.nb_row,
+                                  self.border_mode, self.subsample[0])
+        cols = conv_output_length(cols, self.nb_col,
+                                  self.border_mode, self.subsample[1])
+
+        if self.return_sequences:
+            if self.dim_ordering == 'th':
+                return (input_shape[0], input_shape[1],
+                        self.nb_filter, rows, cols)
+            elif self.dim_ordering == 'tf':
+                return (input_shape[0], input_shape[1],
+                        rows, cols, self.nb_filter)
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+        else:
+            if self.dim_ordering == 'th':
+                return (input_shape[0], self.nb_filter, rows, cols)
+            elif self.dim_ordering == 'tf':
+                return (input_shape[0], rows, cols, self.nb_filter)
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+    def step(self, x, states):
+        raise NotImplementedError
+
+    def get_constants(self, X, train=False):
+        return None
+
+    def get_initial_states(self, X):
+        # (samples, timesteps, row, col, filter)
+        initial_state = K.zeros_like(X)
+        # (samples,row, col, filter)
+        initial_state = K.sum(initial_state, axis=1)
+        initial_state = self.conv_step(initial_state, K.zeros(self.W_shape),
+                                       border_mode=self.border_mode)
+
+        initial_states = [initial_state for _ in range(2)]
+        return initial_states
+
+    def preprocess_input(self, x):
+        return x
+
+    def call(self, x, mask=None):
+        assert K.ndim(x) == 5
+        input_shape = self.input_spec[0].shape
+        unroll = False
+
+        if self.stateful:
+            initial_states = self.states
+        else:
+            initial_states = self.get_initial_states(x)
+
+        constants = self.get_constants(x)
+        preprocessed_input = self.preprocess_input(x)
+
+        last_output, outputs, states = K.rnn(self.step, preprocessed_input,
+                                             initial_states,
+                                             go_backwards=self.go_backwards,
+                                             mask=mask,
+                                             constants=constants,
+                                             unroll=unroll,
+                                             input_length=input_shape[1])
+        if self.stateful:
+            self.updates = []
+            for i in range(len(states)):
+                self.updates.append((self.states[i], states[i]))
+
+        if self.return_sequences:
+            return outputs
+        else:
+            return last_output
+
+    def get_config(self):
+        config = {'return_sequences': self.return_sequences,
+                  'go_backwards': self.go_backwards,
+                  'stateful': self.stateful}
+        if self.stateful:
+            config['batch_input_shape'] = self.input_spec[0].shape
+
+        base_config = super(ConvRecurrent2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class ConvLSTM2D(ConvRecurrent2D):
+    '''Convolutional LSTM.
+
+    # Input shape
+        - if dim_ordering='th'
+            5D tensor with shape:
+            `(samples,time, channels, rows, cols)`
+        - if dim_ordering='tf'
+            5D tensor with shape:
+            `(samples,time, rows, cols, channels)`
+
+     # Output shape
+        - if `return_sequences`
+             - if dim_ordering='th'
+                5D tensor with shape:
+                `(samples, time, nb_filter, output_row, output_col)`
+             - if dim_ordering='tf'
+                5D tensor with shape:
+                `(samples, time, output_row, output_col, nb_filter)`
+        - else
+            - if dim_ordering ='th'
+                4D tensor with shape:
+                `(samples, nb_filter, output_row, output_col)`
+            - if dim_ordering='tf'
+                4D tensor with shape:
+                `(samples, output_row, output_col, nb_filter)`
+
+        where o_row and o_col depend on the shape of the filter and
+        the border_mode
+
+        # Arguments
+            nb_filter: Number of convolution filters to use.
+            nb_row: Number of rows in the convolution kernel.
+            nb_col: Number of columns in the convolution kernel.
+            border_mode: 'valid' or 'same'.
+            sub_sample: tuple of length 2. Factor by which to subsample output.
+                Also called strides elsewhere.
+            dim_ordering: 'tf' if the feature are at the last dimension or 'th'
+            stateful : Boolean (default False). If True, the last state
+                for each sample at index i in a batch will be used as initial
+                state for the sample of index i in the following batch.
+            init: weight initialization function.
+                Can be the name of an existing function (str),
+                or a Theano function
+                (see: [initializations](../initializations.md)).
+            inner_init: initialization function of the inner cells.
+            forget_bias_init: initialization function for the bias of the
+            forget gate.
+                [Jozefowicz et al.](http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf)
+                recommend initializing with ones.
+            activation: activation function.
+                Can be the name of an existing function (str),
+                or a Theano function (see: [activations](../activations.md)).
+            inner_activation: activation function for the inner cells.
+
+    # References
+        - [Convolutional LSTM Network: A Machine Learning Approach for
+        Precipitation Nowcasting](http://arxiv.org/pdf/1506.04214v1.pdf)
+        The current implementation does not include the feedback loop on the
+        cells output
+    '''
+    def __init__(self, nb_filter, nb_row, nb_col,
+                 init='glorot_uniform', inner_init='orthogonal',
+                 forget_bias_init='one', activation='tanh',
+                 inner_activation='hard_sigmoid',
+                 dim_ordering='default',
+                 border_mode='valid', subsample=(1, 1),
+                 W_regularizer=None, U_regularizer=None, b_regularizer=None,
+                 dropout_W=0., dropout_U=0., **kwargs):
+
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        if dim_ordering not in {'tf', 'th'}:
+            raise ValueError('dim_ordering must be in {tf,th}', dim_ordering)
+        self.nb_filter = nb_filter
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.init = initializations.get(init)
+        self.inner_init = initializations.get(inner_init)
+        self.forget_bias_init = initializations.get(forget_bias_init)
+        self.activation = activations.get(activation)
+        self.inner_activation = activations.get(inner_activation)
+        self.border_mode = border_mode
+        self.subsample = subsample
+
+        if dim_ordering == 'th':
+            warnings.warn('Be carefull if used with convolution3D layers:\n'
+                          'th in convolution 3D corresponds to '
+                          '(samples, channels, conv_dim1, conv_dim2,'
+                          'conv_dim3)\n'
+                          'while for this network it corresponds to: '
+                          '(samples, time, channels, rows, cols)')
+        self.dim_ordering = dim_ordering
+
+        kwargs['nb_filter'] = nb_filter
+        kwargs['nb_row'] = nb_row
+        kwargs['nb_col'] = nb_col
+        kwargs['dim_ordering'] = dim_ordering
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.U_regularizer = regularizers.get(U_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.dropout_W, self.dropout_U = dropout_W, dropout_U
+        if self.dropout_W or self.dropout_U:
+            self.uses_learning_phase = True
+
+        super(ConvLSTM2D, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        self.input_spec = [InputSpec(shape=input_shape)]
+
+        if self.dim_ordering == 'th':
+            stack_size = input_shape[2]
+            self.W_shape = (self.nb_filter, stack_size,
+                            self.nb_row, self.nb_col)
+        elif self.dim_ordering == 'tf':
+            stack_size = input_shape[4]
+            self.W_shape = (self.nb_row, self.nb_col,
+                            stack_size, self.nb_filter)
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        if self.dim_ordering == 'th':
+            self.W_shape1 = (self.nb_filter, self.nb_filter,
+                             self.nb_row, self.nb_col)
+        elif self.dim_ordering == 'tf':
+            self.W_shape1 = (self.nb_row, self.nb_col,
+                             self.nb_filter, self.nb_filter)
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        if self.stateful:
+            self.reset_states()
+        else:
+            # initial states: 2 all-zero tensor of shape (nb_filter)
+            self.states = [None, None, None, None]
+
+        self.W_i = self.init(self.W_shape, name='{}_W_i'.format(self.name))
+        self.U_i = self.inner_init(self.W_shape1,
+                                   name='{}_U_i'.format(self.name))
+        self.b_i = K.zeros((self.nb_filter,), name='{}_b_i'.format(self.name))
+
+        self.W_f = self.init(self.W_shape, name='{}_W_f'.format(self.name))
+        self.U_f = self.inner_init(self.W_shape1,
+                                   name='{}_U_f'.format(self.name))
+        self.b_f = self.forget_bias_init((self.nb_filter,),
+                                         name='{}_b_f'.format(self.name))
+
+        self.W_c = self.init(self.W_shape, name='{}_W_c'.format(self.name))
+        self.U_c = self.inner_init(self.W_shape1,
+                                   name='{}_U_c'.format(self.name))
+        self.b_c = K.zeros((self.nb_filter,), name='{}_b_c'.format(self.name))
+
+        self.W_o = self.init(self.W_shape, name='{}_W_o'.format(self.name))
+        self.U_o = self.inner_init(self.W_shape1,
+                                   name='{}_U_o'.format(self.name))
+        self.b_o = K.zeros((self.nb_filter,), name='{}_b_o'.format(self.name))
+
+        self.trainable_weights = [self.W_i, self.U_i, self.b_i,
+                                  self.W_c, self.U_c, self.b_c,
+                                  self.W_f, self.U_f, self.b_f,
+                                  self.W_o, self.U_o, self.b_o]
+
+        self.W = K.concatenate([self.W_i, self.W_f, self.W_c, self.W_o])
+        self.U = K.concatenate([self.U_i, self.U_f, self.U_c, self.U_o])
+        self.b = K.concatenate([self.b_i, self.b_f, self.b_c, self.b_o])
+
+        self.regularizers = []
+        if self.W_regularizer:
+            self.W_regularizer.set_param(self.W)
+            self.regularizers.append(self.W_regularizer)
+        if self.U_regularizer:
+            self.U_regularizer.set_param(self.U)
+            self.regularizers.append(self.U_regularizer)
+        if self.b_regularizer:
+            self.b_regularizer.set_param(self.b)
+            self.regularizers.append(self.b_regularizer)
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+        self.built = True
+
+    def reset_states(self):
+        assert self.stateful, 'Layer must be stateful.'
+        input_shape = self.input_spec[0].shape
+        output_shape = self.get_output_shape_for(input_shape)
+        if not input_shape[0]:
+            raise Exception('If a RNN is stateful, a complete ' +
+                            'input_shape must be provided ' +
+                            '(including batch size).')
+
+        if self.return_sequences:
+            out_row, out_col, out_filter = output_shape[2:]
+        else:
+            out_row, out_col, out_filter = output_shape[1:]
+
+        if hasattr(self, 'states'):
+            K.set_value(self.states[0],
+                        np.zeros((input_shape[0],
+                                  out_row, out_col, out_filter)))
+            K.set_value(self.states[1],
+                        np.zeros((input_shape[0],
+                                  out_row, out_col, out_filter)))
+        else:
+            self.states = [K.zeros((input_shape[0],
+                                    out_row, out_col, out_filter)),
+                           K.zeros((input_shape[0],
+                                    out_row, out_col, out_filter))]
+
+    def conv_step(self, x, W, b=None, border_mode='valid'):
+        input_shape = self.input_spec[0].shape
+
+        conv_out = K.conv2d(x, W, strides=self.subsample,
+                            border_mode=border_mode,
+                            dim_ordering=self.dim_ordering,
+                            image_shape=(input_shape[0],
+                                         input_shape[2],
+                                         input_shape[3],
+                                         input_shape[4]),
+                            filter_shape=self.W_shape)
+        if b:
+            if self.dim_ordering == 'th':
+                conv_out = conv_out + K.reshape(b, (1, self.nb_filter, 1, 1))
+            elif self.dim_ordering == 'tf':
+                conv_out = conv_out + K.reshape(b, (1, 1, 1, self.nb_filter))
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        return conv_out
+
+    def conv_step_hidden(self, x, W, border_mode='valid'):
+        # This new function was defined because the
+        # image shape must be hardcoded
+        input_shape = self.input_spec[0].shape
+        output_shape = self.get_output_shape_for(input_shape)
+        if self.return_sequences:
+            out_row, out_col, out_filter = output_shape[2:]
+        else:
+            out_row, out_col, out_filter = output_shape[1:]
+
+        conv_out = K.conv2d(x, W, strides=(1, 1),
+                            border_mode=border_mode,
+                            dim_ordering=self.dim_ordering,
+                            image_shape=(input_shape[0],
+                                         out_row, out_col,
+                                         out_filter),
+                            filter_shape=self.W_shape1)
+
+        return conv_out
+
+    def step(self, x, states):
+        assert len(states) == 4
+        h_tm1 = states[0]
+        c_tm1 = states[1]
+        B_U = states[2]
+        B_W = states[3]
+
+        x_i = self.conv_step(x * B_W[0], self.W_i, self.b_i,
+                             border_mode=self.border_mode)
+        x_f = self.conv_step(x * B_W[1], self.W_f, self.b_f,
+                             border_mode=self.border_mode)
+        x_c = self.conv_step(x * B_W[2], self.W_c, self.b_c,
+                             border_mode=self.border_mode)
+        x_o = self.conv_step(x * B_W[3], self.W_o, self.b_o,
+                             border_mode=self.border_mode)
+
+        # U : from nb_filter to nb_filter
+        # Same because must be stable in the output space
+        h_i = self.conv_step_hidden(h_tm1 * B_U[0], self.U_i,
+                                    border_mode='same')
+        h_f = self.conv_step_hidden(h_tm1 * B_U[1], self.U_f,
+                                    border_mode='same')
+        h_c = self.conv_step_hidden(h_tm1 * B_U[2], self.U_c,
+                                    border_mode='same')
+        h_o = self.conv_step_hidden(h_tm1 * B_U[3], self.U_o,
+                                    border_mode='same')
+
+        i = self.inner_activation(x_i + h_i)
+        f = self.inner_activation(x_f + h_f)
+        c = f * c_tm1 + i * self.activation(x_c + h_c)
+        o = self.inner_activation(x_o + h_o)
+        h = o * self.activation(c)
+
+        return h, [h, c]
+
+    def get_constants(self, x):
+        constants = []
+        if 0 < self.dropout_U < 1:
+            ones = K.zeros_like(x)
+            ones = K.sum(ones, axis=1)
+            ones = self.conv_step(ones, K.zeros(self.W_shape),
+                                  border_mode=self.border_mode)
+            ones = ones + 1
+            B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones)
+                   for _ in range(4)]
+            constants.append(B_U)
+        else:
+            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
+
+        if 0 < self.dropout_W < 1:
+            ones = K.zeros_like(x)
+            ones = K.sum(ones, axis=1)
+            ones = ones + 1
+            B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones)
+                   for _ in range(4)]
+            constants.append(B_W)
+        else:
+            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
+        return constants
+
+    def get_config(self):
+        config = {'nb_filter': self.nb_filter,
+                  'nb_row': self.nb_row,
+                  'nb_col': self.nb_col,
+                  'init': self.init.__name__,
+                  'inner_init': self.inner_init.__name__,
+                  'forget_bias_init': self.forget_bias_init.__name__,
+                  'activation': self.activation.__name__,
+                  'dim_ordering': self.dim_ordering,
+                  'border_mode': self.border_mode,
+                  'inner_activation': self.inner_activation.__name__}
+        base_config = super(ConvLSTM2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

keras/layers/core.py

@@ -96,6 +96,37 @@ class Dropout(Layer):
         return dict(list(base_config.items()) + list(config.items()))
 
 
+class SpatialDropout1D(Dropout):
+    '''This version performs the same function as Dropout, however it drops
+    entire 1D feature maps instead of individual elements. If adjacent frames
+    within feature maps are strongly correlated (as is normally the case in
+    early convolution layers) then regular dropout will not regularize the
+    activations and will otherwise just result in an effective learning rate
+    decrease. In this case, SpatialDropout1D will help promote independence
+    between feature maps and should be used instead.
+
+    # Arguments
+        p: float between 0 and 1. Fraction of the input units to drop.
+
+    # Input shape
+        3D tensor with shape:
+        `(samples, timesteps, channels)`
+
+    # Output shape
+        Same as input
+
+    # References
+        - [Efficient Object Localization Using Convolutional Networks](https://arxiv.org/pdf/1411.4280.pdf)
+    '''
+    def __init__(self, p, **kwargs):
+        super(SpatialDropout1D, self).__init__(p, **kwargs)
+
+    def _get_noise_shape(self, x):
+        input_shape = K.shape(x)
+        noise_shape = (input_shape[0], 1, input_shape[2])
+        return noise_shape
+    
+    
 class SpatialDropout2D(Dropout):
     '''This version performs the same function as Dropout, however it drops
     entire 2D feature maps instead of individual elements. If adjacent pixels
@@ -111,7 +142,7 @@ class SpatialDropout2D(Dropout):
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -159,7 +190,7 @@ class SpatialDropout3D(Dropout):
             is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         5D tensor with shape:
@@ -661,7 +692,8 @@ class Dense(Layer):
     # Output shape
         2D tensor with shape: `(nb_samples, output_dim)`.
     '''
-    def __init__(self, output_dim, init='glorot_uniform', activation='linear', weights=None,
+    def __init__(self, output_dim, init='glorot_uniform',
+                 activation=None, weights=None,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, input_dim=None, **kwargs):
@@ -722,6 +754,7 @@ class Dense(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def call(self, x, mask=None):
         output = K.dot(x, self.W)
@@ -890,6 +923,7 @@ class MaxoutDense(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         assert input_shape and len(input_shape) == 2
@@ -962,7 +996,7 @@ class Highway(Layer):
         - [Highway Networks](http://arxiv.org/pdf/1505.00387v2.pdf)
     '''
     def __init__(self, init='glorot_uniform', transform_bias=-2,
-                 activation='linear', weights=None,
+                 activation=None, weights=None,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, input_dim=None, **kwargs):
@@ -1027,6 +1061,7 @@ class Highway(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def call(self, x, mask=None):
         y = K.dot(x, self.W_carry)
@@ -1105,7 +1140,7 @@ class TimeDistributedDense(Layer):
     '''
 
     def __init__(self, output_dim,
-                 init='glorot_uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, input_dim=None, input_length=None, **kwargs):
@@ -1167,6 +1202,7 @@ class TimeDistributedDense(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         return (input_shape[0], input_shape[1], self.output_dim)

keras/layers/embeddings.py

@@ -110,6 +110,7 @@ class Embedding(Layer):
 
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
+        self.built = True
 
     def compute_mask(self, x, mask=None):
         if not self.mask_zero:

keras/layers/local.py

@@ -75,7 +75,7 @@ class LocallyConnected1D(Layer):
         `steps` value might have changed due to padding.
     '''
     def __init__(self, nb_filter, filter_length,
-                 init='uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample_length=1,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
@@ -139,6 +139,7 @@ class LocallyConnected1D(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         length = conv_output_length(input_shape[1],
@@ -257,7 +258,7 @@ class LocallyConnected2D(Layer):
         `rows` and `cols` values might have changed due to padding.
     '''
     def __init__(self, nb_filter, nb_row, nb_col,
-                 init='glorot_uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample=(1, 1),
                  dim_ordering='default',
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
@@ -333,6 +334,7 @@ class LocallyConnected2D(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         if self.dim_ordering == 'th':

keras/layers/normalization.py

@@ -58,7 +58,7 @@ class BatchNormalization(Layer):
         Same shape as input.
 
     # References
-        - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](http://jmlr.org/proceedings/papers/v37/ioffe15.html)
+        - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](http://jmlr.org/proceedings/papers/v37/ioffe15.pdf)
     '''
     def __init__(self, epsilon=1e-5, mode=0, axis=-1, momentum=0.99,
                  weights=None, beta_init='zero', gamma_init='one',
@@ -104,7 +104,6 @@ class BatchNormalization(Layer):
             self.set_weights(self.initial_weights)
             del self.initial_weights
         self.built = True
-        self.called_with = None
 
     def call(self, x, mask=None):
         if self.mode == 0 or self.mode == 2:
@@ -122,23 +121,12 @@ class BatchNormalization(Layer):
                     epsilon=self.epsilon)
             else:
                 # mode 0
-                if self.called_with not in {None, x}:
-                    raise Exception('You are attempting to share a '
-                                    'same `BatchNormalization` layer across '
-                                    'different data flows. '
-                                    'This is not possible. '
-                                    'You should use `mode=2` in '
-                                    '`BatchNormalization`, which has '
-                                    'a similar behavior but is shareable '
-                                    '(see docs for a description of '
-                                    'the behavior).')
-                self.called_with = x
                 x_normed, mean, std = K.normalize_batch_in_training(
                     x, self.gamma, self.beta, reduction_axes,
                     epsilon=self.epsilon)
 
-                self.updates = [K.moving_average_update(self.running_mean, mean, self.momentum),
-                                K.moving_average_update(self.running_std, std, self.momentum)]
+                self.add_updates([K.moving_average_update(self.running_mean, mean, self.momentum),
+                                  K.moving_average_update(self.running_std, std, self.momentum)], x)
 
                 if K.backend() == 'tensorflow' and sorted(reduction_axes) == range(K.ndim(x))[:-1]:
                     x_normed_running = K.batch_normalization(
@@ -168,11 +156,11 @@ class BatchNormalization(Layer):
         return x_normed
 
     def get_config(self):
-        config = {"epsilon": self.epsilon,
-                  "mode": self.mode,
-                  "axis": self.axis,
-                  "gamma_regularizer": self.gamma_regularizer.get_config() if self.gamma_regularizer else None,
-                  "beta_regularizer": self.beta_regularizer.get_config() if self.beta_regularizer else None,
-                  "momentum": self.momentum}
+        config = {'epsilon': self.epsilon,
+                  'mode': self.mode,
+                  'axis': self.axis,
+                  'gamma_regularizer': self.gamma_regularizer.get_config() if self.gamma_regularizer else None,
+                  'beta_regularizer': self.beta_regularizer.get_config() if self.beta_regularizer else None,
+                  'momentum': self.momentum}
         base_config = super(BatchNormalization, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))

keras/layers/pooling.py

@@ -34,14 +34,12 @@ class _Pooling1D(Layer):
         raise NotImplementedError
 
     def call(self, x, mask=None):
-        x = K.expand_dims(x, -1)   # add dummy last dimension
-        x = K.permute_dimensions(x, (0, 2, 1, 3))
+        x = K.expand_dims(x, 2)   # add dummy last dimension
         output = self._pooling_function(inputs=x, pool_size=self.pool_size,
                                         strides=self.st,
                                         border_mode=self.border_mode,
-                                        dim_ordering='th')
-        output = K.permute_dimensions(output, (0, 2, 1, 3))
-        return K.squeeze(output, 3)  # remove dummy last dimension
+                                        dim_ordering='tf')
+        return K.squeeze(output, 2)  # remove dummy last dimension
 
     def get_config(self):
         config = {'stride': self.stride,
@@ -66,7 +64,6 @@ class MaxPooling1D(_Pooling1D):
             2 will halve the input.
             If None, it will default to `pool_length`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
     '''
 
     def __init__(self, pool_length=2, stride=None,
@@ -89,7 +86,6 @@ class AveragePooling1D(_Pooling1D):
         stride: integer, or None. Stride value.
             If None, it will default to `pool_length`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
 
     # Input shape
         3D tensor with shape: `(samples, steps, features)`.
@@ -181,12 +177,11 @@ class MaxPooling2D(_Pooling2D):
         strides: tuple of 2 integers, or None. Strides values.
             If None, it will default to `pool_size`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -223,12 +218,11 @@ class AveragePooling2D(_Pooling2D):
         strides: tuple of 2 integers, or None. Strides values.
             If None, it will default to `pool_size`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -333,7 +327,7 @@ class MaxPooling3D(_Pooling3D):
             (the depth) is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         5D tensor with shape:
@@ -373,7 +367,7 @@ class AveragePooling3D(_Pooling3D):
             (the depth) is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         5D tensor with shape:
@@ -447,7 +441,6 @@ class _GlobalPooling2D(Layer):
         super(_GlobalPooling2D, self).__init__(**kwargs)
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
-        print(dim_ordering)
         self.dim_ordering = dim_ordering
         self.input_spec = [InputSpec(ndim=4)]
 
@@ -474,7 +467,7 @@ class GlobalAveragePooling2D(_GlobalPooling2D):
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -502,7 +495,7 @@ class GlobalMaxPooling2D(_GlobalPooling2D):
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -520,3 +513,83 @@ class GlobalMaxPooling2D(_GlobalPooling2D):
             return K.max(x, axis=[1, 2])
         else:
             return K.max(x, axis=[2, 3])
+
+
+class _GlobalPooling3D(Layer):
+
+    def __init__(self, dim_ordering='default', **kwargs):
+        super(_GlobalPooling3D, self).__init__(**kwargs)
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        self.dim_ordering = dim_ordering
+        self.input_spec = [InputSpec(ndim=5)]
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'tf':
+            return (input_shape[0], input_shape[4])
+        else:
+            return (input_shape[0], input_shape[1])
+
+    def call(self, x, mask=None):
+        raise NotImplementedError
+
+    def get_config(self):
+        config = {'dim_ordering': self.dim_ordering}
+        base_config = super(_GlobalPooling3D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class GlobalAveragePooling3D(_GlobalPooling3D):
+    '''Global Average pooling operation for 3D data.
+
+    # Arguments
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+
+    # Input shape
+        5D tensor with shape:
+        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        2D tensor with shape:
+        `(nb_samples, channels)`
+    '''
+
+    def call(self, x, mask=None):
+        if self.dim_ordering == 'tf':
+            return K.mean(x, axis=[1, 2, 3])
+        else:
+            return K.mean(x, axis=[2, 3, 4])
+
+
+class GlobalMaxPooling3D(_GlobalPooling3D):
+    '''Global Max pooling operation for 3D data.
+
+    # Arguments
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+
+    # Input shape
+        5D tensor with shape:
+        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        2D tensor with shape:
+        `(nb_samples, channels)`
+    '''
+
+    def call(self, x, mask=None):
+        if self.dim_ordering == 'tf':
+            return K.max(x, axis=[1, 2, 3])
+        else:
+            return K.max(x, axis=[2, 3, 4])

keras/layers/recurrent.py

@@ -199,6 +199,18 @@ class Recurrent(Layer):
         # note that the .build() method of subclasses MUST define
         # self.input_spec with a complete input shape.
         input_shape = self.input_spec[0].shape
+        if self.unroll and input_shape[1] is None:
+            raise ValueError('Cannot unroll a RNN if the '
+                             'time dimension is undefined. \n'
+                             '- If using a Sequential model, '
+                             'specify the time dimension by passing '
+                             'an `input_shape` or `batch_input_shape` '
+                             'argument to your first layer. If your '
+                             'first layer is an Embedding, you can '
+                             'also use the `input_length` argument.\n'
+                             '- If using the functional API, specify '
+                             'the time dimension by passing a `shape` '
+                             'or `batch_shape` argument to your Input layer.')
         if self.stateful:
             initial_states = self.states
         else:
@@ -214,9 +226,10 @@ class Recurrent(Layer):
                                              unroll=self.unroll,
                                              input_length=input_shape[1])
         if self.stateful:
-            self.updates = []
+            updates = []
             for i in range(len(states)):
-                self.updates.append((self.states[i], states[i]))
+                updates.append((self.states[i], states[i]))
+            self.add_updates(updates, x)
 
         if self.return_sequences:
             return outputs
@@ -229,7 +242,7 @@ class Recurrent(Layer):
                   'stateful': self.stateful,
                   'unroll': self.unroll,
                   'consume_less': self.consume_less}
-        if self.stateful:
+        if self.stateful and self.input_spec[0].shape:
             config['batch_input_shape'] = self.input_spec[0].shape
         else:
             config['input_dim'] = self.input_dim
@@ -313,13 +326,22 @@ class SimpleRNN(Recurrent):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def reset_states(self):
         assert self.stateful, 'Layer must be stateful.'
         input_shape = self.input_spec[0].shape
         if not input_shape[0]:
-            raise Exception('If a RNN is stateful, a complete ' +
-                            'input_shape must be provided (including batch size).')
+            raise Exception('If a RNN is stateful, it needs to know '
+                            'its batch size. Specify the batch size '
+                            'of your input tensors: \n'
+                            '- If using a Sequential model, '
+                            'specify the batch size by passing '
+                            'a `batch_input_shape` '
+                            'argument to your first layer.\n'
+                            '- If using the functional API, specify '
+                            'the time dimension by passing a '
+                            '`batch_shape` argument to your Input layer.')
         if hasattr(self, 'states'):
             K.set_value(self.states[0],
                         np.zeros((input_shape[0], self.output_dim)))
@@ -363,7 +385,7 @@ class SimpleRNN(Recurrent):
             input_shape = self.input_spec[0].shape
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.tile(ones, (1, input_dim))
+            ones = K.tile(ones, (1, int(input_dim)))
             B_W = K.in_train_phase(K.dropout(ones, self.dropout_W), ones)
             constants.append(B_W)
         else:
@@ -495,6 +517,7 @@ class GRU(Recurrent):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def reset_states(self):
         assert self.stateful, 'Layer must be stateful.'
@@ -577,7 +600,7 @@ class GRU(Recurrent):
             input_shape = self.input_spec[0].shape
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.tile(ones, (1, input_dim))
+            ones = K.tile(ones, (1, int(input_dim)))
             B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(3)]
             constants.append(B_W)
         else:
@@ -725,6 +748,7 @@ class LSTM(Recurrent):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def reset_states(self):
         assert self.stateful, 'Layer must be stateful.'
@@ -817,7 +841,7 @@ class LSTM(Recurrent):
             input_shape = self.input_spec[0].shape
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.tile(ones, (1, input_dim))
+            ones = K.tile(ones, (1, int(input_dim)))
             B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(4)]
             constants.append(B_W)
         else:

keras/layers/wrappers.py

@@ -113,8 +113,10 @@ class TimeDistributed(Wrapper):
                 output = self.layer.call(x)
                 return output, []
 
-            last_output, outputs, states = K.rnn(step, X,
-                                                 initial_states=[])
+            _, outputs, _ = K.rnn(step, X,
+                                  initial_states=[],
+                                  input_length=input_shape[1],
+                                  unroll=False)
             y = outputs
         else:
             # no batch size specified, therefore the layer will be able

keras/legacy/models.py

@@ -1,775 +0,0 @@
-from collections import OrderedDict
-import warnings
-import copy
-
-from .. import backend as K
-from ..layers import InputLayer, Layer, Merge
-from ..engine.training import Model
-
-
-class Graph(Model):
-    '''Arbitrary connection graph.
-
-    THIS IS A LEGACY MODEL AND SHOULD NOT BE USED
-    except for backwards compatibility support.
-
-    For multi-inputs/multi-outputs models, or
-    models using shared layers, use the functional API instead.
-    '''
-
-    def __init__(self, name=None):
-        # model attributes
-        self.inbound_nodes = []
-        self.outbound_nodes = []
-        self.built = False
-        self.supports_masking = False
-
-        # legacy attributes (we prefix them with _graph_)
-        self._graph_namespace = set()  # strings
-        self._graph_nodes = OrderedDict()  # layer-like
-        self._graph_inputs = OrderedDict()  # layer-like
-        self._graph_outputs = OrderedDict()  # layer-like
-        self._graph_input_config = []  # dicts
-        self._graph_output_config = []  # dicts
-        self._graph_node_config = []  # dicts
-        self._graph_shared_nodes_names = []
-
-        if not name:
-            prefix = 'graph_'
-            name = prefix + str(K.get_uid(prefix))
-        self.name = name
-
-    def __call__(self, x, mask=None):
-        self.build()
-        return super(Graph, self).__call__(x, mask)
-
-    def build(self, input_shape=None):
-        # this will crash if the input/output layers have multiple nodes
-        # no plans to support that case since Graph is deprecated
-        input_tensors = [layer.output for layer in self._graph_inputs.values()]
-        output_tensors = [layer.output for layer in self._graph_outputs.values()]
-        # actually create the model
-        super(Graph, self).__init__(input_tensors,
-                                    output_tensors,
-                                    name=self.name)
-        self.built = True
-
-    def compile(self, optimizer, loss,
-                metrics=[],
-                sample_weight_modes=None,
-                loss_weights=None,
-                **kwargs):
-        '''Configures the learning process.
-
-        # Arguments
-            optimizer: str (name of optimizer) or optimizer object.
-                See [optimizers](optimizers.md).
-            loss: dictionary mapping the name(s) of the output(s) to
-                a loss function (string name of objective function or
-                objective function. See [objectives](objectives.md)).
-            metrics: list of str (name of metrics) or
-                list of metrics functions. See [metrics](metrics.md).
-            sample_weight_modes: optional dictionary mapping certain
-                output names to a sample weight mode ("temporal" and None
-                are the only supported modes). If you need to do
-                timestep-wise loss weighting on one of your graph outputs,
-                you will need to set the sample weight mode for this output
-                to "temporal".
-            loss_weights: dictionary you can pass to specify a weight
-                coefficient for each loss function (in a multi-output model).
-                If no loss weight is specified for an output,
-                the weight for this output's loss will be considered to be 1.
-            kwargs: for Theano backend, these are passed into K.function.
-                Ignored for Tensorflow backend.
-        '''
-        # create the underlying Model
-        if not self.built:
-            self.build()
-        super(Graph, self).compile(optimizer, loss,
-                                   metrics=metrics,
-                                   sample_weight_mode=sample_weight_modes,
-                                   loss_weights=loss_weights,
-                                   **kwargs)
-
-    def add_input(self, name, input_shape=None,
-                  batch_input_shape=None, dtype='float'):
-        '''Adds an input to the graph.
-
-        # Arguments:
-            name: string. The name of the new input.
-                Must be unique in the graph.
-            input_shape: a tuple of integers,
-                the expected shape of the input samples.
-                Does not include the batch size.
-            batch_input_shape: a tuple of integers,
-                the expected shape of the whole input batch,
-                including the batch size.
-            dtype: 'float', or 'int'.
-        '''
-        if name in self._graph_namespace:
-            raise Exception('Duplicate node identifier: ' + name)
-        self._graph_namespace.add(name)
-        self.built = False
-
-        if dtype[:3] == 'int':
-            dtype = 'int32'
-        elif dtype[:5] == 'float':
-            dtype = K.floatx()
-        else:
-            raise Exception('Uknown dtype (should be "int" or "float"): ' +
-                            str(dtype))
-
-        # create input layer
-        input_layer = InputLayer(input_shape=input_shape,
-                                 batch_input_shape=batch_input_shape,
-                                 name=name, input_dtype=dtype)
-        self._graph_inputs[name] = input_layer
-
-        # append input config to self._graph_input_config
-        config = {'name': name, 'dtype': dtype}
-        if batch_input_shape:
-            config['batch_input_shape'] = batch_input_shape
-        else:
-            config['input_shape'] = input_shape
-        self._graph_input_config.append(config)
-
-    def add_node(self, layer, name, input=None, inputs=[],
-                 merge_mode='concat', concat_axis=-1, dot_axes=-1,
-                 create_output=False):
-        '''Adds a node in the graph. It can be connected to multiple
-        inputs, which will first be merged into one tensor
-        according to the mode specified.
-
-        # Arguments
-            layer: the layer at the node.
-            name: name for the node.
-            input: when connecting the layer to a single input,
-                this is the name of the incoming node.
-            inputs: when connecting the layer to multiple inputs,
-                this is a list of names of incoming nodes.
-            merge_mode: one of {concat, sum, dot, ave, mul}
-            concat_axis: when `merge_mode=='concat'`, this is the
-                input concatenation axis.
-            dot_axes: when `merge_mode='dot'`,
-                this is the contraction axes specification;
-                see the `Merge` layer for details.
-            create_output: boolean. Set this to `True` if you want the output
-                of your node to be an output of the graph.
-        '''
-        if name in self._graph_namespace:
-            raise Exception('Duplicate node identifier: ' + name)
-        self._graph_namespace.add(name)
-        layer.name = name
-        self.built = False
-
-        if input:
-            if input not in self._graph_namespace:
-                raise Exception('Unknown node/input identifier: ' + input)
-            if input in self._graph_nodes:
-                layer.add_inbound_node(self._graph_nodes[input])
-            elif input in self._graph_inputs:
-                layer.add_inbound_node(self._graph_inputs[input])
-        if inputs:
-            to_merge = []
-            for n in inputs:
-                if n in self._graph_nodes:
-                    to_merge.append(self._graph_nodes[n])
-                elif n in self._graph_inputs:
-                    to_merge.append(self._graph_inputs[n])
-                else:
-                    raise Exception('Unknown identifier: ' + n)
-            merge = Merge(to_merge, mode=merge_mode,
-                          concat_axis=concat_axis, dot_axes=dot_axes,
-                          name='merge_inputs_for_' + name)
-            layer.add_inbound_node(merge)
-        self._graph_nodes[name] = layer
-        self._graph_node_config.append({'name': name,
-                                        'input': input,
-                                        'inputs': inputs,
-                                        'merge_mode': merge_mode,
-                                        'concat_axis': concat_axis,
-                                        'dot_axes': dot_axes,
-                                        'create_output': create_output})
-        if create_output:
-            self.add_output(name, input=name)
-
-    def add_shared_node(self, layer, name, inputs=[], merge_mode=None,
-                        concat_axis=-1, dot_axes=-1, outputs=[],
-                        create_output=False):
-        '''Used to share a same layer across multiple nodes.
-
-        Supposed, for instance, that you want to apply one same `Dense` layer
-        after two different nodes ('node_a' and 'node_b').
-        You can then add the dense layer as a shared node by calling:
-
-        ```python
-        model.add_shared_node(my_dense, name='shared_dense', inputs=['node_a', 'node_b'], ...)
-        ```
-
-        If you want access to the output of dense(node_a) and dense(node_b) separately,
-        you can add these outputs to the Graph by passing an `outputs` argument:
-
-        ```python
-        model.add_shared_node(my_dense, name='shared_dense', inputs=['node_a', 'node_b'],
-                              outputs=['dense_output_a', 'dense_outputs_b'])
-        ```
-
-        Otherwise you can merge these different outputs via `merge_mode`.
-        In that case you can access the merged output
-        under the identifier `name`.
-
-        # Arguments
-            layer: The layer to be shared across multiple inputs
-            name: Name of the shared node
-            inputs: List of names of input nodes
-            merge_mode: Same meaning as `merge_mode` argument of `add_node()`
-            concat_axis: Same meaning as `concat_axis` argument of `add_node()`
-            dot_axes: Same meaning as `dot_axes` argument of `add_node()`
-            outputs: Used when `merge_mode=None`. Names for the output nodes.
-            create_output: Same meaning as `create_output` argument of `add_node()`.
-        '''
-        if name in self._graph_namespace:
-            raise Exception('Duplicate node identifier: ' + name)
-        self._graph_namespace.add(name)
-        self.built = False
-
-        for o in outputs:
-            if o in self._graph_namespace:
-                raise Exception('Duplicate node identifier: ' + o)
-        if merge_mode:
-            if merge_mode not in {'sum', 'ave', 'mul', 'dot', 'cos', 'concat'}:
-                raise Exception('Invalid merge mode:', merge_mode)
-        input_layers = []
-        for i in range(len(inputs)):
-            input = inputs[i]
-            if input in self._graph_nodes:
-                n = self._graph_nodes[input]
-                input_layers.append(n)
-            elif input in self._graph_inputs:
-                n = self._graph_inputs[input]
-                input_layers.append(n)
-            else:
-                raise Exception('Unknown identifier: ' + input)
-
-        created_node_indices = []
-        for input_layer in input_layers:
-            created_node_indices.append(len(layer.inbound_nodes))
-            layer.add_inbound_node(input_layer)
-
-        if merge_mode:
-            layer.name = 'input_for_' + name
-            # collect all output nodes of layer and merge them into a single output
-            merge = Merge([layer for _ in range(len(inputs))],
-                          mode=merge_mode,
-                          concat_axis=concat_axis, dot_axes=dot_axes,
-                          node_indices=created_node_indices,
-                          name=name)
-            self._graph_nodes[name] = merge
-            if create_output:
-                self.add_output(name, input=name)
-        else:
-            layer.name = name
-            # create one new layer per output node of layer,
-            # and add them to the Graph with their own identifiers
-            if len(outputs) != len(inputs):
-                raise Exception('When using merge_mode=None, '
-                                'you should provide a list of '
-                                'output names (`output` argument) '
-                                'the same size as `input`.')
-            for i in range(len(outputs)):
-                output_layer_name = outputs[i]
-                output_layer = Layer(name=output_layer_name)
-                output_layer.add_inbound_node(layer, created_node_indices[i])
-                self._graph_namespace.add(output_layer_name)
-                self._graph_nodes[output_layer_name] = output_layer
-                if create_output:
-                    self.add_output(output_layer_name, input=output_layer_name)
-
-        self._graph_node_config.append({'name': name,
-                                        'layer': {
-                                            'config': layer.get_config(),
-                                            'class_name': layer.__class__.__name__,
-                                        },
-                                        'inputs': inputs,
-                                        'merge_mode': merge_mode,
-                                        'concat_axis': concat_axis,
-                                        'dot_axes': dot_axes,
-                                        'outputs': outputs,
-                                        'create_output': create_output if merge_mode else False})
-        self._graph_shared_nodes_names.append(name)
-
-    def add_output(self, name, input=None, inputs=[],
-                   merge_mode='concat', concat_axis=-1, dot_axes=-1):
-        '''Adds an output to the graph.
-
-        This output can merge several node outputs into a single output.
-
-        # Arguments
-            name: name of the output.
-            input: when connecting the layer to a single input,
-                this is the name of the incoming node.
-            inputs: when connecting the layer to multiple inputs,
-                this is a list of names of incoming nodes.
-            merge_mode: one of {concat, sum, dot, ave, mul}
-            concat_axis: when `merge_mode=='concat'`, this is the
-                input concatenation axis.
-            dot_axes: when `merge_mode='dot'`,
-                this is the contraction axes specification;
-                see the `Merge layer for details.
-        '''
-        if name not in self._graph_namespace:
-            self._graph_namespace.add(name)
-        if name in self._graph_outputs:
-            raise Exception('Duplicate output identifier:', name)
-        self.built = False
-
-        if input:
-            if input in self._graph_nodes:
-                layer = self._graph_nodes[input]
-            elif input in self._graph_inputs:
-                layer = self._graph_inputs[input]
-            else:
-                raise Exception('Unknown node/input identifier: ' + input)
-            if layer.name == name:
-                self._graph_outputs[name] = layer
-            else:
-                layer.name = name
-                self._graph_outputs[name] = layer
-        if inputs:
-            to_merge = []
-            for n in inputs:
-                if n not in self._graph_nodes:
-                    raise Exception('Unknown identifier: ' + n)
-                to_merge.append(self._graph_nodes[n])
-            merge = Merge(to_merge, mode=merge_mode,
-                          concat_axis=concat_axis, dot_axes=dot_axes,
-                          name=name)
-            self._graph_outputs[name] = merge
-
-        self._graph_output_config.append({'name': name,
-                                          'input': input,
-                                          'inputs': inputs,
-                                          'merge_mode': merge_mode,
-                                          'concat_axis': concat_axis,
-                                          'dot_axes': dot_axes})
-
-    def _get_x(self, data):
-        x = []
-        for key in self._graph_inputs.keys():
-            if key not in data:
-                raise Exception('Expected to be provided an array '
-                                '(in dict argument `data`) for input "' +
-                                key + '".')
-            x.append(data[key])
-        return x
-
-    def _get_y(self, data):
-        y = []
-        for key in self._graph_outputs.keys():
-            if key not in data:
-                raise Exception('Expected to be provided an array '
-                                '(in dict argument `data`) for output "' +
-                                key + '".')
-            y.append(data[key])
-        return y
-
-    def fit(self, data, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
-            validation_split=0., validation_data=None, shuffle=True,
-            class_weight=None, sample_weight=None, **kwargs):
-        '''Trains the model for a fixed number of epochs.
-
-        Returns a history object. Its `history` attribute is a record of
-        training loss values at successive epochs,
-        as well as validation loss values (if applicable).
-
-        # Arguments
-            data: dictionary mapping input names and outputs names to
-                appropriate Numpy arrays. All arrays should contain
-                the same number of samples.
-            batch_size: int. Number of samples per gradient update.
-            nb_epoch: int.
-            verbose: 0 for no logging to stdout,
-                1 for progress bar logging, 2 for one log line per epoch.
-            callbacks: `keras.callbacks.Callback` list. List of callbacks
-                to apply during training. See [callbacks](callbacks.md).
-            validation_split: float (0. < x < 1). Fraction of the data to
-                use as held-out validation data.
-            validation_data: dictionary mapping input names and outputs names
-                to appropriate Numpy arrays to be used as
-                held-out validation data.
-                All arrays should contain the same number of samples.
-                Will override validation_split.
-            shuffle: boolean. Whether to shuffle the samples at each epoch.
-            class_weight: dictionary mapping output names to
-                class weight dictionaries.
-            sample_weight: dictionary mapping output names to
-                numpy arrays of sample weights.
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-
-        if type(validation_data) is tuple:
-            raise Exception('Cannot used sample_weight with '
-                            'validation data with legacy Graph model. '
-                            'validation_data should be a dictionary.')
-        if validation_data:
-            val_x = self._get_x(validation_data)
-            val_y = self._get_y(validation_data)
-            validation_data = (val_x, val_y)
-        return super(Graph, self).fit(x, y,
-                                      batch_size=batch_size,
-                                      nb_epoch=nb_epoch,
-                                      verbose=verbose,
-                                      callbacks=callbacks,
-                                      validation_split=validation_split,
-                                      validation_data=validation_data,
-                                      shuffle=shuffle,
-                                      class_weight=class_weight,
-                                      sample_weight=sample_weight)
-
-    def evaluate(self, data, batch_size=128,
-                 verbose=0, sample_weight={}, **kwargs):
-        '''Computes the loss on some input data, batch by batch.
-
-        Returns the scalar test loss over the data,
-        or a list of metrics values (starting with the test loss)
-        if applicable.
-
-        Arguments: see `fit` method.
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-        return super(Graph, self).evaluate(x, y,
-                                           batch_size=batch_size,
-                                           verbose=verbose,
-                                           sample_weight=sample_weight)
-
-    def predict(self, data, batch_size=128, verbose=0):
-        '''Generates output predictions for the input samples
-        batch by batch.
-
-        Arguments: see `fit` method.
-        '''
-        x = self._get_x(data)
-        output_list = super(Graph, self).predict(x, batch_size=batch_size,
-                                                 verbose=verbose)
-        if not isinstance(output_list, list):
-            output_list = [output_list]
-        return dict(zip(self._graph_outputs, output_list))
-
-    def train_on_batch(self, data,
-                       class_weight={},
-                       sample_weight={}, **kwargs):
-        '''Single gradient update on a batch of samples.
-
-        Returns the scalar train loss over the data,
-        or a list of metrics values (starting with the test loss)
-        if applicable.
-
-        Arguments: see `fit` method.
-        '''
-        if 'accuracy' in kwargs:
-            kwargs.pop('accuracy')
-            warnings.warn('The "accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-        return super(Graph, self).train_on_batch(x, y,
-                                                 sample_weight=sample_weight,
-                                                 class_weight=class_weight)
-
-    def test_on_batch(self, data, sample_weight={}, **kwargs):
-        '''Test the network on a single batch of samples.
-
-        Returns the scalar test loss over the data,
-        or a list of metrics values (starting with the test loss)
-        if applicable.
-
-        Arguments: see `fit` method.
-        '''
-        if 'accuracy' in kwargs:
-            kwargs.pop('accuracy')
-            warnings.warn('The "accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-        return super(Graph, self).test_on_batch(x, y,
-                                                sample_weight=sample_weight)
-
-    def predict_on_batch(self, data):
-        output_list = super(Graph, self).predict_on_batch(data)
-        if not isinstance(output_list, list):
-            output_list = [output_list]
-        return dict(zip(self._graph_outputs, output_list))
-
-    def fit_generator(self, generator, samples_per_epoch, nb_epoch,
-                      verbose=1, callbacks=[],
-                      validation_data=None, nb_val_samples=None,
-                      class_weight={},
-                      max_q_size=10, **kwargs):
-        '''Fits a model on data generated batch-by-batch by a Python generator.
-        The generator is run in parallel to the model, for efficiency.
-        For instance, this allows you to do real-time data augmentation
-        on images on CPU in parallel to training your model on GPU.
-
-        # Arguments
-            generator: a generator.
-                The output of the generator must be either a tuple
-                of dictionaries `(input_data, sample_weight)`
-                or a dictionary `input_data`
-                (mapping names of inputs and outputs to Numpy arrays).
-                All arrays should contain the same number of samples.
-                The generator is expected to loop over its data
-                indefinitely. An epoch finishes when `samples_per_epoch`
-                samples have been seen by the model.
-            samples_per_epoch: integer, number of samples to process before
-                going to the next epoch.
-            nb_epoch: integer, total number of iterations on the data.
-            verbose: verbosity mode, 0, 1, or 2.
-            callbacks: list of callbacks to be called during training.
-            validation_data: dictionary mapping input names and outputs names
-                to appropriate Numpy arrays to be used as
-                held-out validation data, or a generator yielding such
-                dictionaries. All arrays should contain the same number
-                of samples. If a generator, will be called until more than
-                `nb_val_samples` examples have been generated at the
-                end of every epoch. These examples will then be used
-                as the validation data.
-            nb_val_samples: number of samples to use from validation
-                generator at the end of every epoch.
-            class_weight: dictionary mapping class indices to a weight
-                for the class.
-
-        # Returns
-            A `History` object.
-
-        # Examples
-
-        ```python
-            def generate_arrays_from_file(path):
-                while 1:
-                    f = open(path)
-                    for line in f:
-                        # create Numpy arrays of input data
-                        # and labels, from each line in the file
-                        x1, x2, y = process_line(line)
-                        yield ({'input_1': x1, 'input_2': x2, 'output': y})
-                    f.close()
-
-            graph.fit_generator(generate_arrays_from_file('/my_file.txt'),
-                                samples_per_epoch=10000, nb_epoch=10)
-        ```
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if 'nb_worker' in kwargs:
-            kwargs.pop('nb_worker')
-            warnings.warn('The "nb_worker" argument is deprecated, '
-                          'please remove it from your code.')
-        if 'nb_val_worker' in kwargs:
-            kwargs.pop('nb_val_worker')
-            warnings.warn('The "nb_val_worker" argument is deprecated, '
-                          'please remove it from your code.')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-
-        self._train_on_batch = self.train_on_batch
-        self.train_on_batch = super(Graph, self).train_on_batch
-        self._evaluate = self.evaluate
-        self.evaluate = super(Graph, self).evaluate
-
-        if validation_data and type(validation_data) is tuple:
-            raise Exception('Cannot use sample_weight with '
-                            'validation_data in legacy Graph model.')
-        if validation_data and type(validation_data) is dict:
-            validation_data = (self._get_x(validation_data),
-                               self._get_y(validation_data))
-
-        original_generator = generator
-
-        def fixed_generator():
-            while 1:
-                data = next(original_generator)
-                if type(data) is tuple:
-                    data, sample_weight = data
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y, sample_weight
-                else:
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y
-
-        generator = fixed_generator()
-        history = super(Graph, self).fit_generator(generator,
-                                                   samples_per_epoch,
-                                                   nb_epoch,
-                                                   verbose=verbose,
-                                                   callbacks=callbacks,
-                                                   validation_data=validation_data,
-                                                   nb_val_samples=nb_val_samples,
-                                                   class_weight=class_weight,
-                                                   max_q_size=max_q_size)
-        self.train_on_batch = self._train_on_batch
-        self.evaluate = self._evaluate
-        return history
-
-    def evaluate_generator(self, generator, val_samples,
-                           verbose=1, max_q_size=10, **kwargs):
-        '''Evaluates the model on a generator. The generator should
-        return the same kind of data with every yield as accepted
-        by `evaluate`.
-
-        If `show_accuracy`, it returns a tuple `(loss, accuracy)`,
-        otherwise it returns the loss value.
-
-        Arguments:
-            generator:
-                generator yielding dictionaries of the kind accepted
-                by `evaluate`, or tuples of such dictionaries and
-                associated dictionaries of sample weights.
-            val_samples:
-                total number of samples to generate from `generator`
-                to use in validation.
-
-            Other arguments are the same as for `fit`.
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if 'verbose' in kwargs:
-            kwargs.pop('verbose')
-            warnings.warn('The "verbose" argument is deprecated.')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-
-        self._test_on_batch = self.test_on_batch
-        self.test_on_batch = super(Graph, self).test_on_batch
-
-        original_generator = generator
-
-        def fixed_generator():
-            while 1:
-                data = next(original_generator)
-                if type(data) is tuple:
-                    data, sample_weight = data
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y, sample_weight
-                else:
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y
-
-        generator = fixed_generator()
-        history = super(Graph, self).evaluate_generator(generator,
-                                                        val_samples,
-                                                        max_q_size=max_q_size)
-        self.test_on_batch = self._test_on_batch
-        return history
-
-    # get_weights, set_weights: inherited
-    def get_config(self):
-        config = {'input_config': self._graph_input_config,
-                  'node_config': self._graph_node_config,
-                  'output_config': self._graph_output_config}
-        nodes = {}
-        for name, node in self._graph_nodes.items():
-            nodes[name] = {'class_name': node.__class__.__name__,
-                           'config': node.get_config()}
-            if name in self._graph_shared_nodes_names:
-                nodes[name]['shared'] = True
-        config['nodes'] = nodes
-        return copy.deepcopy(config)
-
-    @classmethod
-    def from_config(cls, config):
-        # TODO: test legacy support
-        from keras.utils.layer_utils import layer_from_config
-
-        def normalize_legacy_config(conf):
-            if 'class_name' not in conf:
-                class_name = conf['name']
-                name = conf.get('custom_name')
-                conf['name'] = name
-                new_config = {
-                    'class_name': class_name,
-                    'config': conf,
-                }
-                return new_config
-            return conf
-
-        graph = cls()
-        inputs = config.get('input_config')
-        for input in inputs:
-            graph.add_input(**input)
-
-        nodes = config.get('node_config')
-        for node in nodes:
-            layer_config = config['nodes'][node['name']]
-            layer_config = normalize_legacy_config(layer_config)
-            if 'layer' in node:
-                # for add_shared_node
-                node['layer'] = layer_from_config(node['layer'])
-            else:
-                layer = layer_from_config(layer_config)
-                node['layer'] = layer
-
-            node['create_output'] = False  # outputs will be added below
-            if layer_config.get('shared'):
-                graph.add_shared_node(**node)
-            else:
-                graph.add_node(**node)
-
-        outputs = config.get('output_config')
-        for output in outputs:
-            graph.add_output(**output)
-        return graph
-
-    def load_weights(self, fname):
-        if not self.built:
-            self.build()
-        super(Graph, self).load_weights(fname)

keras/metrics.py

@@ -1,78 +1,134 @@
 import numpy as np
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def binary_accuracy(y_true, y_pred):
+    '''Calculates the mean accuracy rate across all predictions for binary
+    classification problems.
+    '''
     return K.mean(K.equal(y_true, K.round(y_pred)))
 
 
 def categorical_accuracy(y_true, y_pred):
+    '''Calculates the mean accuracy rate across all predictions for
+    multiclass classification problems.
+    '''
     return K.mean(K.equal(K.argmax(y_true, axis=-1),
                   K.argmax(y_pred, axis=-1)))
 
 
 def sparse_categorical_accuracy(y_true, y_pred):
+    '''Same as categorical_accuracy, but useful when the predictions are for
+    sparse targets.
+    '''
     return K.mean(K.equal(K.max(y_true, axis=-1),
                           K.cast(K.argmax(y_pred, axis=-1), K.floatx())))
 
 
+def top_k_categorical_accuracy(y_true, y_pred, k=5):
+    '''Calculates the top-k categorical accuracy rate, i.e. success when the
+    target class is within the top-k predictions provided.
+    '''
+    return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k))
+
+
 def mean_squared_error(y_true, y_pred):
+    '''Calculates the mean squared error (mse) rate
+    between predicted and target values.
+    '''
     return K.mean(K.square(y_pred - y_true))
 
 
 def mean_absolute_error(y_true, y_pred):
+    '''Calculates the mean absolute error (mae) rate
+    between predicted and target values.
+    '''
     return K.mean(K.abs(y_pred - y_true))
 
 
 def mean_absolute_percentage_error(y_true, y_pred):
+    '''Calculates the mean absolute percentage error (mape) rate
+    between predicted and target values.
+    '''
     diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), np.inf))
     return 100. * K.mean(diff)
 
 
 def mean_squared_logarithmic_error(y_true, y_pred):
+    '''Calculates the mean squared logarithmic error (msle) rate
+    between predicted and target values.
+    '''
     first_log = K.log(K.clip(y_pred, K.epsilon(), np.inf) + 1.)
     second_log = K.log(K.clip(y_true, K.epsilon(), np.inf) + 1.)
     return K.mean(K.square(first_log - second_log))
 
 
-def squared_hinge(y_true, y_pred):
-    return K.mean(K.square(K.maximum(1. - y_true * y_pred, 0.)))
-
-
 def hinge(y_true, y_pred):
+    '''Calculates the hinge loss, which is defined as
+    `max(1 - y_true * y_pred, 0)`.
+    '''
     return K.mean(K.maximum(1. - y_true * y_pred, 0.))
 
 
+def squared_hinge(y_true, y_pred):
+    '''Calculates the squared value of the hinge loss.
+    '''
+    return K.mean(K.square(K.maximum(1. - y_true * y_pred, 0.)))
+
+
 def categorical_crossentropy(y_true, y_pred):
-    '''Expects a binary class matrix instead of a vector of scalar classes.
+    '''Calculates the cross-entropy value for multiclass classification
+    problems. Note: Expects a binary class matrix instead of a vector
+    of scalar classes.
     '''
     return K.mean(K.categorical_crossentropy(y_pred, y_true))
 
 
 def sparse_categorical_crossentropy(y_true, y_pred):
-    '''expects an array of integer classes.
-    Note: labels shape must have the same number of dimensions as output shape.
-    If you get a shape error, add a length-1 dimension to labels.
+    '''Calculates the cross-entropy value for multiclass classification
+    problems with sparse targets. Note: Expects an array of integer
+    classes. Labels shape must have the same number of dimensions as
+    output shape. If you get a shape error, add a length-1 dimension
+    to labels.
     '''
     return K.mean(K.sparse_categorical_crossentropy(y_pred, y_true))
 
 
 def binary_crossentropy(y_true, y_pred):
+    '''Calculates the cross-entropy value for binary classification
+    problems.
+    '''
     return K.mean(K.binary_crossentropy(y_pred, y_true))
 
 
+def kullback_leibler_divergence(y_true, y_pred):
+    '''Calculates the Kullback-Leibler (KL) divergence between prediction
+    and target values.
+    '''
+    y_true = K.clip(y_true, K.epsilon(), 1)
+    y_pred = K.clip(y_pred, K.epsilon(), 1)
+    return K.sum(y_true * K.log(y_true / y_pred), axis=-1)
+
+
 def poisson(y_true, y_pred):
+    '''Calculates the poisson function over prediction and target values.
+    '''
     return K.mean(y_pred - y_true * K.log(y_pred + K.epsilon()))
 
 
 def cosine_proximity(y_true, y_pred):
+    '''Calculates the cosine similarity between the prediction and target
+    values.
+    '''
     y_true = K.l2_normalize(y_true, axis=-1)
     y_pred = K.l2_normalize(y_pred, axis=-1)
     return -K.mean(y_true * y_pred)
 
 
 def matthews_correlation(y_true, y_pred):
-    ''' Matthews correlation coefficient
+    '''Calculates the Matthews correlation coefficient measure for quality
+    of binary classification problems.
     '''
     y_pred_pos = K.round(K.clip(y_pred, 0, 1))
     y_pred_neg = 1 - y_pred_pos
@@ -83,22 +139,78 @@ def matthews_correlation(y_true, y_pred):
     tp = K.sum(y_pos * y_pred_pos)
     tn = K.sum(y_neg * y_pred_neg)
 
-    fp = K.sum(1 - y_neg * y_pred_pos)
-    fn = K.sum(1 - y_pos * y_pred_neg)
-    
+    fp = K.sum(y_neg * y_pred_pos)
+    fn = K.sum(y_pos * y_pred_neg)
+
     numerator = (tp * tn - fp * fn)
     denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
 
     return numerator / (denominator + K.epsilon())
 
+
+def precision(y_true, y_pred):
+    '''Calculates the precision, a metric for multi-label classification of
+    how many selected items are relevant.
+    '''
+    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
+    precision = true_positives / (predicted_positives + K.epsilon())
+    return precision
+
+
+def recall(y_true, y_pred):
+    '''Calculates the recall, a metric for multi-label classification of
+    how many relevant items are selected.
+    '''
+    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
+    recall = true_positives / (possible_positives + K.epsilon())
+    return recall
+
+
+def fbeta_score(y_true, y_pred, beta=1):
+    '''Calculates the F score, the weighted harmonic mean of precision and recall.
+
+    This is useful for multi-label classification, where input samples can be
+    classified as sets of labels. By only using accuracy (precision) a model
+    would achieve a perfect score by simply assigning every class to every
+    input. In order to avoid this, a metric should penalize incorrect class
+    assignments as well (recall). The F-beta score (ranged from 0.0 to 1.0)
+    computes this, as a weighted mean of the proportion of correct class
+    assignments vs. the proportion of incorrect class assignments.
+
+    With beta = 1, this is equivalent to a F-measure. With beta < 1, assigning
+    correct classes becomes more important, and with beta > 1 the metric is
+    instead weighted towards penalizing incorrect class assignments.
+    '''
+    if beta < 0:
+        raise ValueError('The lowest choosable beta is zero (only precision).')
+        
+    # If there are no true positives, fix the F score at 0 like sklearn.
+    if K.sum(K.round(K.clip(y_true, 0, 1))) == 0:
+        return 0
+
+    p = precision(y_true, y_pred)
+    r = recall(y_true, y_pred)
+    bb = beta ** 2
+    fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())
+    return fbeta_score
+
+
+def fmeasure(y_true, y_pred):
+    '''Calculates the f-measure, the harmonic mean of precision and recall.
+    '''
+    return fbeta_score(y_true, y_pred, beta=1)
+
+
 # aliases
 mse = MSE = mean_squared_error
 mae = MAE = mean_absolute_error
 mape = MAPE = mean_absolute_percentage_error
 msle = MSLE = mean_squared_logarithmic_error
 cosine = cosine_proximity
+fscore = f1score = fmeasure
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier):
     return get_from_module(identifier, globals(), 'metric')

keras/models.py

@@ -6,11 +6,11 @@ import os
 import numpy as np
 
 from . import backend as K
+from . import optimizers
 from .utils.io_utils import ask_to_proceed_with_overwrite
 from .engine.training import Model
-from .engine.topology import get_source_inputs, Node
+from .engine.topology import get_source_inputs, Node, Layer, Merge
 from .optimizers import optimizer_from_config
-from .legacy.models import Graph
 
 
 def save_model(model, filepath, overwrite=True):
@@ -56,40 +56,52 @@ def save_model(model, filepath, overwrite=True):
     model.save_weights_to_hdf5_group(model_weights_group)
 
     if hasattr(model, 'optimizer'):
-        f.attrs['training_config'] = json.dumps({
-            'optimizer_config': {
-                'class_name': model.optimizer.__class__.__name__,
-                'config': model.optimizer.get_config()
-            },
-            'loss': model.loss,
-            'metrics': model.metrics,
-            'sample_weight_mode': model.sample_weight_mode,
-            'loss_weights': model.loss_weights,
-        }, default=get_json_type).encode('utf8')
+        if isinstance(model.optimizer, optimizers.TFOptimizer):
+            warnings.warn(
+                'TensorFlow optimizers do not '
+                'make it possible to access '
+                'optimizer attributes or optimizer state '
+                'after instantiation. '
+                'As a result, we cannot save the optimizer '
+                'as part of the model save file.'
+                'You will have to compile your model again after loading it. '
+                'Prefer using a Keras optimizer instead '
+                '(see keras.io/optimizers).')
+        else:
+            f.attrs['training_config'] = json.dumps({
+                'optimizer_config': {
+                    'class_name': model.optimizer.__class__.__name__,
+                    'config': model.optimizer.get_config()
+                },
+                'loss': model.loss,
+                'metrics': model.metrics,
+                'sample_weight_mode': model.sample_weight_mode,
+                'loss_weights': model.loss_weights,
+            }, default=get_json_type).encode('utf8')
 
-        # save optimizer weights
-        symbolic_weights = getattr(model.optimizer, 'weights')
-        if symbolic_weights:
-            optimizer_weights_group = f.create_group('optimizer_weights')
-            weight_values = K.batch_get_value(symbolic_weights)
-            weight_names = []
-            for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
-                if hasattr(w, 'name') and w.name:
-                    name = str(w.name)
-                else:
-                    name = 'param_' + str(i)
-                weight_names.append(name.encode('utf8'))
-            optimizer_weights_group.attrs['weight_names'] = weight_names
-            for name, val in zip(weight_names, weight_values):
-                param_dset = optimizer_weights_group.create_dataset(
-                    name,
-                    val.shape,
-                    dtype=val.dtype)
-                if not val.shape:
-                    # scalar
-                    param_dset[()] = val
-                else:
-                    param_dset[:] = val
+            # save optimizer weights
+            symbolic_weights = getattr(model.optimizer, 'weights')
+            if symbolic_weights:
+                optimizer_weights_group = f.create_group('optimizer_weights')
+                weight_values = K.batch_get_value(symbolic_weights)
+                weight_names = []
+                for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
+                    if hasattr(w, 'name') and w.name:
+                        name = str(w.name)
+                    else:
+                        name = 'param_' + str(i)
+                    weight_names.append(name.encode('utf8'))
+                optimizer_weights_group.attrs['weight_names'] = weight_names
+                for name, val in zip(weight_names, weight_values):
+                    param_dset = optimizer_weights_group.create_dataset(
+                        name,
+                        val.shape,
+                        dtype=val.dtype)
+                    if not val.shape:
+                        # scalar
+                        param_dset[()] = val
+                    else:
+                        param_dset[:] = val
     f.flush()
     f.close()
 
@@ -157,7 +169,7 @@ def load_model(filepath, custom_objects={}):
     # set optimizer weights
     if 'optimizer_weights' in f:
         # build train function (to get weight updates)
-        if model.__class__.__name__ == 'Sequential':
+        if isinstance(model, Sequential):
             model.model._make_train_function()
         else:
             model._make_train_function()
@@ -238,7 +250,7 @@ class Sequential(Model):
         self.model = None  # internal Model instance
         self.inputs = []  # tensors
         self.outputs = []  # tensors (length 1)
-        self.trainable = True
+        self._trainable = True
 
         # model attributes
         self.inbound_nodes = []
@@ -260,6 +272,10 @@ class Sequential(Model):
         # Arguments
             layer: layer instance.
         '''
+        if not isinstance(layer, Layer):
+            raise ValueError('The added layer must be '
+                             'an instance of class Layer. '
+                             'Found: ' + str(layer))
         if not self.outputs:
             # first layer in model: check that it is an input layer
             if len(layer.inbound_nodes) == 0:
@@ -367,6 +383,7 @@ class Sequential(Model):
                             ' Add some layers first.')
         # actually create the model
         self.model = Model(self.inputs, self.outputs[0], name=self.name + '_model')
+        self.model.trainable = self.trainable
 
         # mirror model attributes
         self.supports_masking = self.model.supports_masking
@@ -400,26 +417,27 @@ class Sequential(Model):
         if self._flattened_layers is not None:
             return self._flattened_layers
         layers = []
-        if self.layers[0].__class__.__name__ == 'Merge':
-            merge = self.layers[0]
-            for layer in merge.layers:
-                if hasattr(layer, 'flattened_layers'):
-                    for sublayer in layer.flattened_layers:
-                        if sublayer not in layers:
-                            layers.append(sublayer)
-                elif hasattr(layer, 'layers'):
-                    for sublayer in layer.layers:
-                        if sublayer not in layers:
-                            layers.append(sublayer)
-                else:
-                    if layer not in layers:
-                        layers.append(layer)
-        else:
-            if self.layers[0] not in layers:
-                layers.append(self.layers[0])
-        for layer in self.layers[1:]:
-            if layer not in layers:
-                layers.append(layer)
+        if self.layers:
+            if isinstance(self.layers[0], Merge):
+                merge = self.layers[0]
+                for layer in merge.layers:
+                    if hasattr(layer, 'flattened_layers'):
+                        for sublayer in layer.flattened_layers:
+                            if sublayer not in layers:
+                                layers.append(sublayer)
+                    elif hasattr(layer, 'layers'):
+                        for sublayer in layer.layers:
+                            if sublayer not in layers:
+                                layers.append(sublayer)
+                    else:
+                        if layer not in layers:
+                            layers.append(layer)
+            else:
+                if self.layers[0] not in layers:
+                    layers.append(self.layers[0])
+            for layer in self.layers[1:]:
+                if layer not in layers:
+                    layers.append(layer)
         self._flattened_layers = layers
         return layers
 
@@ -437,6 +455,16 @@ class Sequential(Model):
                              list(layer_dict.items()))
         return all_attrs
 
+    @property
+    def trainable(self):
+        return self._trainable
+
+    @trainable.setter
+    def trainable(self, value):
+        if self.model:
+            self.model.trainable = value
+        self._trainable = value
+
     @property
     def trainable_weights(self):
         if not self.trainable:
@@ -455,13 +483,15 @@ class Sequential(Model):
 
     @property
     def updates(self):
-        # support for legacy behavior
-        return self._gather_list_attr('updates')
+        return self.model.updates
 
     @property
     def state_updates(self):
         # support for legacy behavior
-        return self._gather_list_attr('state_updates')
+        return self.model.state_updates
+
+    def get_updates_for(self, inputs):
+        return self.model.get_updates_for(inputs)
 
     @property
     def regularizers(self):
@@ -517,6 +547,7 @@ class Sequential(Model):
             metrics: list of metrics to be evaluated by the model
                 during training and testing.
                 Typically you will use `metrics=['accuracy']`.
+                See [metrics](/metrics).
             sample_weight_mode: if you need to do timestep-wise
                 sample weighting (2D weights), set this to "temporal".
                 "None" defaults to sample-wise weights (1D).
@@ -571,7 +602,8 @@ class Sequential(Model):
                 See [callbacks](/callbacks).
             validation_split: float (0. < x < 1).
                 Fraction of the data to use as held-out validation data.
-            validation_data: tuple (X, y) to be used as held-out
+            validation_data: tuple (x_val, y_val) or tuple
+                (x_val, y_val, val_sample_weights) to be used as held-out
                 validation data. Will override validation_split.
             shuffle: boolean or str (for 'batch').
                 Whether to shuffle the samples at each epoch.
@@ -785,7 +817,8 @@ class Sequential(Model):
     def fit_generator(self, generator, samples_per_epoch, nb_epoch,
                       verbose=1, callbacks=[],
                       validation_data=None, nb_val_samples=None,
-                      class_weight=None, max_q_size=10, nb_worker=1, pickle_safe=False, **kwargs):
+                      class_weight=None, max_q_size=10, nb_worker=1,
+                      pickle_safe=False, **kwargs):
         '''Fits the model on data generated batch-by-batch by
         a Python generator.
         The generator is run in parallel to the model, for efficiency.
@@ -873,7 +906,9 @@ class Sequential(Model):
                                         nb_worker=nb_worker,
                                         pickle_safe=pickle_safe)
 
-    def evaluate_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False, **kwargs):
+    def evaluate_generator(self, generator, val_samples,
+                           max_q_size=10, nb_worker=1,
+                           pickle_safe=False, **kwargs):
         '''Evaluates the model on a data generator. The generator should
         return the same kind of data as accepted by `test_on_batch`.
 
@@ -915,7 +950,8 @@ class Sequential(Model):
                                              nb_worker=nb_worker,
                                              pickle_safe=pickle_safe)
 
-    def predict_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False):
+    def predict_generator(self, generator, val_samples,
+                          max_q_size=10, nb_worker=1, pickle_safe=False):
         '''Generates predictions for the input samples from a data generator.
         The generator should return the same kind of data as accepted by
         `predict_on_batch`.
@@ -949,7 +985,7 @@ class Sequential(Model):
         as a Python list.
         '''
         config = []
-        if self.layers[0].__class__.__name__ == 'Merge':
+        if isinstance(self.layers[0], Merge):
             assert hasattr(self.layers[0], 'layers')
             layers = []
             for layer in self.layers[0].layers:

keras/objectives.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import numpy as np
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def mean_squared_error(y_true, y_pred):
@@ -72,6 +73,6 @@ msle = MSLE = mean_squared_logarithmic_error
 kld = KLD = kullback_leibler_divergence
 cosine = cosine_proximity
 
-from .utils.generic_utils import get_from_module
+
 def get(identifier):
     return get_from_module(identifier, globals(), 'objective')

keras/optimizers.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import
 from . import backend as K
 from .utils.generic_utils import get_from_module
 from six.moves import zip
+import warnings
 
 
 def clip_norm(g, c, n):
@@ -19,6 +20,7 @@ def optimizer_from_config(config, custom_objects={}):
         'adam': Adam,
         'adamax': Adamax,
         'nadam': Nadam,
+        'tfoptimizer': TFOptimizer,
     }
     class_name = config['class_name']
     if class_name in custom_objects:
@@ -53,14 +55,6 @@ class Optimizer(object):
         self.updates = []
         self.weights = []
 
-    def get_state(self):
-        return [K.get_value(u[0]) for u in self.updates]
-
-    def set_state(self, value_list):
-        assert len(self.updates) == len(value_list)
-        for u, v in zip(self.updates, value_list):
-            K.set_value(u[0], v)
-
     def get_updates(self, params, constraints, loss):
         raise NotImplementedError
 
@@ -230,6 +224,7 @@ class RMSprop(Optimizer):
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
                   'rho': float(K.get_value(self.rho)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(RMSprop, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -281,6 +276,7 @@ class Adagrad(Optimizer):
 
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adagrad, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -346,6 +342,7 @@ class Adadelta(Optimizer):
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
                   'rho': self.rho,
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adadelta, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -411,6 +408,7 @@ class Adam(Optimizer):
         config = {'lr': float(K.get_value(self.lr)),
                   'beta_1': float(K.get_value(self.beta_1)),
                   'beta_2': float(K.get_value(self.beta_2)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adam, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -450,7 +448,7 @@ class Adamax(Optimizer):
             lr *= (1. / (1. + self.decay * self.iterations))
 
         t = self.iterations + 1
-        lr_t = self.lr / (1. - K.pow(self.beta_1, t))
+        lr_t = lr / (1. - K.pow(self.beta_1, t))
 
         shapes = [K.get_variable_shape(p) for p in params]
         # zero init of 1st moment
@@ -480,6 +478,7 @@ class Adamax(Optimizer):
         config = {'lr': float(K.get_value(self.lr)),
                   'beta_1': float(K.get_value(self.beta_1)),
                   'beta_2': float(K.get_value(self.beta_2)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adamax, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -565,6 +564,36 @@ class Nadam(Optimizer):
         return dict(list(base_config.items()) + list(config.items()))
 
 
+class TFOptimizer(Optimizer):
+
+    def __init__(self, optimizer):
+        self.optimizer = optimizer
+        self.iterations = K.variable(0.)
+        self.updates = []
+
+    def get_updates(self, params, constraints, loss):
+        if constraints:
+            raise ValueError('TF optimizers do not support '
+                             'weights constraints. Either remove '
+                             'all weights constraints in your model, '
+                             'or use a Keras optimizer.')
+        grads = self.optimizer.compute_gradients(loss, params)
+        opt_update = self.optimizer.apply_gradients(
+            grads, global_step=self.iterations)
+        self.updates.append(opt_update)
+        return self.updates
+
+    @property
+    def weights(self):
+        raise NotImplementedError
+
+    def get_config(self):
+        raise NotImplementedError
+
+    def from_config(self, config):
+        raise NotImplementedError
+
+
 # aliases
 sgd = SGD
 rmsprop = RMSprop
@@ -576,5 +605,11 @@ nadam = Nadam
 
 
 def get(identifier, kwargs=None):
+    if K.backend() == 'tensorflow':
+        # Wrap TF optimizer instances
+        import tensorflow as tf
+        if isinstance(identifier, tf.train.Optimizer):
+            return TFOptimizer(identifier)
+    # Instantiate a Keras optimizer
     return get_from_module(identifier, globals(), 'optimizer',
                            instantiate=True, kwargs=kwargs)

keras/preprocessing/image.py

@@ -181,7 +181,7 @@ def load_img(path, grayscale=False, target_size=None):
 
 
 def list_pictures(directory, ext='jpg|jpeg|bmp|png'):
-    return [os.path.join(directory, f) for f in os.listdir(directory)
+    return [os.path.join(directory, f) for f in sorted(os.listdir(directory))
             if os.path.isfile(os.path.join(directory, f)) and re.match('([\w]+\.(?:' + ext + '))', f)]
 
 
@@ -390,6 +390,9 @@ class ImageDataGenerator(object):
                 how many augmentation passes to do over the data
             seed: random seed.
         '''
+        if seed is not None:
+            np.random.seed(seed)
+
         X = np.copy(X)
         if augment:
             aX = np.zeros(tuple([rounds * X.shape[0]] + list(X.shape)[1:]))
@@ -408,7 +411,7 @@ class ImageDataGenerator(object):
 
         if self.zca_whitening:
             flatX = np.reshape(X, (X.shape[0], X.shape[1] * X.shape[2] * X.shape[3]))
-            sigma = np.dot(flatX.T, flatX) / flatX.shape[1]
+            sigma = np.dot(flatX.T, flatX) / flatX.shape[0]
             U, S, V = linalg.svd(sigma)
             self.principal_components = np.dot(np.dot(U, np.diag(1. / np.sqrt(S + 10e-7))), U.T)
 
@@ -431,11 +434,11 @@ class Iterator(object):
         # ensure self.batch_index is 0
         self.reset()
         while 1:
+            if seed is not None:
+                np.random.seed(seed + self.total_batches_seen)
             if self.batch_index == 0:
                 index_array = np.arange(N)
                 if shuffle:
-                    if seed is not None:
-                        np.random.seed(seed + self.total_batches_seen)
                     index_array = np.random.permutation(N)
 
             current_index = (self.batch_index * batch_size) % N
@@ -560,7 +563,7 @@ class DirectoryIterator(Iterator):
 
         for subdir in classes:
             subpath = os.path.join(directory, subdir)
-            for fname in os.listdir(subpath):
+            for fname in sorted(os.listdir(subpath)):
                 is_valid = False
                 for extension in white_list_formats:
                     if fname.lower().endswith('.' + extension):
@@ -576,7 +579,7 @@ class DirectoryIterator(Iterator):
         i = 0
         for subdir in classes:
             subpath = os.path.join(directory, subdir)
-            for fname in os.listdir(subpath):
+            for fname in sorted(os.listdir(subpath)):
                 is_valid = False
                 for extension in white_list_formats:
                     if fname.lower().endswith('.' + extension):

keras/utils/data_utils.py

@@ -40,6 +40,20 @@ else:
 
 def get_file(fname, origin, untar=False,
              md5_hash=None, cache_subdir='datasets'):
+    '''Downloads a file from a URL if it not already in the cache.
+
+    Passing the MD5 hash will verify the file after download as well as if it is already present in the cache.
+
+    # Arguments
+        fname: name of the file
+        origin: original URL of the file
+        untar: boolean, whether the file should be decompressed
+        md5_hash: MD5 hash of the file for verification
+        cache_subdir: directory being used as the cache
+
+    # Returns
+        Path to the downloaded file
+    '''
     datadir_base = os.path.expanduser(os.path.join('~', '.keras'))
     if not os.access(datadir_base, os.W_OK):
         datadir_base = os.path.join('/tmp', '.keras')
@@ -110,6 +124,15 @@ def get_file(fname, origin, untar=False,
 
 
 def validate_file(fpath, md5_hash):
+    '''Validates a file against a MD5 hash
+
+    # Arguments
+        fpath: path to the file being validated
+        md5_hash: the MD5 hash being validated against
+
+    # Returns
+        Whether the file is valid
+    '''
     hasher = hashlib.md5()
     with open(fpath, 'rb') as f:
         buf = f.read()

keras/utils/generic_utils.py

@@ -66,7 +66,7 @@ def func_reconstruct_closure(values):
     src += ["  return lambda:(%s)" % ','.join(["_%d" % n for n in nums]), ""]
     src = '\n'.join(src)
     try:
-        exec(src)
+        exec(src, globals())
     except:
         raise SyntaxError(src)
     return func(values).__closure__

keras/utils/io_utils.py

@@ -6,9 +6,33 @@ from collections import defaultdict
 
 
 class HDF5Matrix():
+    '''Representation of HDF5 dataset which can be used instead of a
+    Numpy array.
+
+    # Example
+
+    ```python
+        X_data = HDF5Matrix('input/file.hdf5', 'data')
+        model.predict(X_data)
+    ```
+
+    Providing start and end allows use of a slice of the dataset.
+
+    Optionally, a normalizer function (or lambda) can be given. This will
+    be called on every slice of data retrieved.
+
+    # Arguments
+        datapath: string, path to a HDF5 file
+        dataset: string, name of the HDF5 dataset in the file specified
+            in datapath
+        start: int, start of desired slice of the specified dataset
+        end: int, end of desired slice of the specified dataset
+        normalizer: function to be called on data when retrieved
+
+    '''
     refs = defaultdict(int)
 
-    def __init__(self, datapath, dataset, start, end, normalizer=None):
+    def __init__(self, datapath, dataset, start=0, end=None, normalizer=None):
         import h5py
 
         if datapath not in list(self.refs.keys()):
@@ -16,9 +40,12 @@ class HDF5Matrix():
             self.refs[datapath] = f
         else:
             f = self.refs[datapath]
-        self.start = start
-        self.end = end
         self.data = f[dataset]
+        self.start = start
+        if end is None:
+            self.end = self.data.shape[0]
+        else:
+            self.end = end
         self.normalizer = normalizer
 
     def __len__(self):

keras/utils/layer_utils.py

@@ -3,7 +3,7 @@ from __future__ import print_function
 from .generic_utils import get_from_module
 from .np_utils import convert_kernel
 from ..layers import *
-from ..models import Model, Sequential, Graph
+from ..models import Model, Sequential
 from .. import backend as K
 
 
@@ -15,7 +15,7 @@ def layer_from_config(config, custom_objects={}):
             of custom (non-Keras) objects to class/functions
 
     # Returns
-        Layer instance (may be Model, Sequential, Graph, Layer...)
+        Layer instance (may be Model, Sequential, Layer...)
     '''
     # Insert custom layers into globals so they can
     # be accessed by `get_from_module`.
@@ -26,8 +26,6 @@ def layer_from_config(config, custom_objects={}):
 
     if class_name == 'Sequential':
         layer_class = Sequential
-    elif class_name == 'Graph':
-        layer_class = Graph
     elif class_name in ['Model', 'Container']:
         layer_class = Model
     else:
@@ -37,8 +35,14 @@ def layer_from_config(config, custom_objects={}):
 
 
 def print_summary(layers, relevant_nodes=None, line_length=100, positions=[.33, .55, .67, 1.]):
-    # line_length: total length of printed lines
-    # positions: relative or absolute positions of log elements in each line
+    '''Prints a summary of a layer
+
+    # Arguments
+        layers: list of layers to print summaries of
+        relevant_nodes: list of relevant nodes
+        line_length: total length of printed lines
+        positions: relative or absolute positions of log elements in each line
+    '''
     if positions[-1] <= 1:
         positions = [int(line_length * p) for p in positions]
     # header names for the different log elements
@@ -47,6 +51,8 @@ def print_summary(layers, relevant_nodes=None, line_length=100, positions=[.33,
     def print_row(fields, positions):
         line = ''
         for i in range(len(fields)):
+            if i > 0:
+                line = line[:-1] + ' '
             line += str(fields[i])
             line = line[:positions[i]]
             line += ' ' * (positions[i] - len(line))
@@ -87,16 +93,28 @@ def print_summary(layers, relevant_nodes=None, line_length=100, positions=[.33,
                 fields = ['', '', '', connections[i]]
                 print_row(fields, positions)
 
-    total_params = 0
     for i in range(len(layers)):
         print_layer_summary(layers[i])
         if i == len(layers) - 1:
             print('=' * line_length)
         else:
             print('_' * line_length)
-        total_params += layers[i].count_params()
 
-    print('Total params: %s' % total_params)
+    def count_total_params(layers, layer_set=None):
+        if layer_set is None:
+            layer_set = set()
+        total_params = 0
+        for layer in layers:
+            if layer in layer_set:
+                continue
+            layer_set.add(layer)
+            if type(layer) in (Model, Sequential):
+                total_params += count_total_params(layer.layers, layer_set)
+            else:
+                total_params += layer.count_params()
+        return total_params
+
+    print('Total params: %s' % count_total_params(layers))
     print('_' * line_length)
 
 

keras/utils/np_utils.py

@@ -3,11 +3,18 @@ import numpy as np
 import scipy as sp
 from six.moves import range
 from six.moves import zip
+from .. import backend as K
 
 
 def to_categorical(y, nb_classes=None):
-    '''Convert class vector (integers from 0 to nb_classes)
-    to binary class matrix, for use with categorical_crossentropy.
+    '''Convert class vector (integers from 0 to nb_classes) to binary class matrix, for use with categorical_crossentropy.
+
+    # Arguments
+        y: class vector to be converted into a matrix
+        nb_classes: total number of classes
+
+    # Returns
+        A binary matrix representation of the input.
     '''
     if not nb_classes:
         nb_classes = np.max(y)+1
@@ -52,12 +59,14 @@ def categorical_probas_to_classes(p):
     return np.argmax(p, axis=1)
 
 
-def convert_kernel(kernel, dim_ordering='th'):
+def convert_kernel(kernel, dim_ordering='default'):
     '''Converts a kernel matrix (Numpy array)
     from Theano format to TensorFlow format
     (or reciprocally, since the transformation
     is its own inverse).
     '''
+    if dim_ordering == 'default':
+        dim_ordering = K.image_dim_ordering()
     new_kernel = np.copy(kernel)
     if kernel.ndim == 4:
         # conv 2d
@@ -113,21 +122,25 @@ def convert_kernel(kernel, dim_ordering='th'):
 def conv_output_length(input_length, filter_size, border_mode, stride, dilation=1):
     if input_length is None:
         return None
-    assert border_mode in {'same', 'valid'}
+    assert border_mode in {'same', 'valid', 'full'}
     dilated_filter_size = filter_size + (filter_size - 1) * (dilation - 1)
     if border_mode == 'same':
         output_length = input_length
     elif border_mode == 'valid':
         output_length = input_length - dilated_filter_size + 1
+    elif border_mode == 'full':
+        output_length = input_length + dilated_filter_size - 1
     return (output_length + stride - 1) // stride
 
 
 def conv_input_length(output_length, filter_size, border_mode, stride):
     if output_length is None:
         return None
-    assert border_mode in {'same', 'valid'}
+    assert border_mode in {'same', 'valid', 'full'}
     if border_mode == 'same':
         pad = filter_size // 2
     elif border_mode == 'valid':
         pad = 0
+    elif border_mode == 'full':
+        pad = filter_size - 1
     return (output_length - 1) * stride - 2 * pad + filter_size

keras/utils/test_utils.py

@@ -1,7 +1,7 @@
 import numpy as np
 from numpy.testing import assert_allclose
 import inspect
-import functools
+import six
 
 from ..engine import Model, Input
 from ..models import Sequential, model_from_json
@@ -112,7 +112,7 @@ def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None,
 def keras_test(func):
     '''Clean up after tensorflow tests.
     '''
-    @functools.wraps(func)
+    @six.wraps(func)
     def wrapper(*args, **kwargs):
         output = func(*args, **kwargs)
         if K._BACKEND == 'tensorflow':

keras/utils/visualize_util.py

@@ -1,3 +1,8 @@
+import os
+
+from ..layers.wrappers import Wrapper
+from ..models import Sequential
+
 try:
     # pydot-ng is a fork of pydot that is better maintained
     import pydot_ng as pydot
@@ -15,23 +20,32 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True):
     dot.set('concentrate', True)
     dot.set_node_defaults(shape='record')
 
-    if model.__class__.__name__ == 'Sequential':
+    if isinstance(model, Sequential):
         if not model.built:
             model.build()
         model = model.model
     layers = model.layers
 
-    # first, populate the nodes of the graph
+    # Create graph nodes.
     for layer in layers:
         layer_id = str(id(layer))
-        if show_layer_names:
-            label = str(layer.name) + ' (' + layer.__class__.__name__ + ')'
-        else:
-            label = layer.__class__.__name__
 
+        # Append a wrapped layer's label to node's label, if it exists.
+        layer_name = layer.name
+        class_name = layer.__class__.__name__
+        if isinstance(layer, Wrapper):
+            layer_name = '{}({})'.format(layer_name, layer.layer.name)
+            child_class_name = layer.layer.__class__.__name__
+            class_name = '{}({})'.format(class_name, child_class_name)
+
+        # Create node's label.
+        if show_layer_names:
+            label = '{}: {}'.format(layer_name, class_name)
+        else:
+            label = class_name
+
+        # Rebuild the label as a table including input/output shapes.
         if show_shapes:
-            # Build the label that will actually contain a table with the
-            # input/output
             try:
                 outputlabels = str(layer.output_shape)
             except:
@@ -48,13 +62,12 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True):
         node = pydot.Node(layer_id, label=label)
         dot.add_node(node)
 
-    # second, add the edges
+    # Connect nodes with edges.
     for layer in layers:
         layer_id = str(id(layer))
         for i, node in enumerate(layer.inbound_nodes):
             node_key = layer.name + '_ib-' + str(i)
             if node_key in model.container_nodes:
-                # add edges
                 for inbound_layer in node.inbound_layers:
                     inbound_layer_id = str(id(inbound_layer))
                     layer_id = str(id(layer))
@@ -64,4 +77,9 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True):
 
 def plot(model, to_file='model.png', show_shapes=False, show_layer_names=True):
     dot = model_to_dot(model, show_shapes, show_layer_names)
-    dot.write_png(to_file)
+    _, format = os.path.splitext(to_file)
+    if not format:
+        format = 'png'
+    else:
+        format = format[1:]
+    dot.write(to_file, format=format)

keras/wrappers/scikit_learn.py

@@ -66,7 +66,7 @@ class BaseWrapper(object):
                             Sequential.predict_classes, Sequential.evaluate]
         if self.build_fn is None:
             legal_params_fns.append(self.__call__)
-        elif not isinstance(self.build_fn, types.FunctionType):
+        elif not isinstance(self.build_fn, types.FunctionType) and not isinstance(self.build_fn, types.MethodType):
             legal_params_fns.append(self.build_fn.__call__)
         else:
             legal_params_fns.append(self.build_fn)
@@ -130,7 +130,7 @@ class BaseWrapper(object):
 
         if self.build_fn is None:
             self.model = self.__call__(**self.filter_sk_params(self.__call__))
-        elif not isinstance(self.build_fn, types.FunctionType):
+        elif not isinstance(self.build_fn, types.FunctionType) and not isinstance(self.build_fn, types.MethodType):
             self.model = self.build_fn(
                 **self.filter_sk_params(self.build_fn.__call__))
         else:

setup.py

@@ -3,12 +3,12 @@ from setuptools import find_packages
 
 
 setup(name='Keras',
-      version='1.1.0',
+      version='1.1.2',
       description='Deep Learning for Python',
       author='Francois Chollet',
       author_email='francois.chollet@gmail.com',
       url='https://github.com/fchollet/keras',
-      download_url='https://github.com/fchollet/keras/tarball/1.1.0',
+      download_url='https://github.com/fchollet/keras/tarball/1.1.2',
       license='MIT',
       install_requires=['theano', 'pyyaml', 'six'],
       extras_require={

tests/keras/backend/test_backends.py

@@ -492,6 +492,7 @@ class TestBackend(object):
         check_single_tensor_operation('relu', (4, 2), alpha=0.1, max_value=0.5)
         check_single_tensor_operation('softmax', (4, 10))
         check_single_tensor_operation('softplus', (4, 10))
+        check_single_tensor_operation('elu', (4, 10), alpha=0.5)
 
         check_single_tensor_operation('sigmoid', (4, 2))
         check_single_tensor_operation('hard_sigmoid', (4, 2))
@@ -528,7 +529,7 @@ class TestBackend(object):
 
                 kernel_val = np.random.random(kernel_shape) - 0.5
 
-                kernel_th = KTH.variable(convert_kernel(kernel_val))
+                kernel_th = KTH.variable(convert_kernel(kernel_val, dim_ordering='th'))
                 kernel_tf = KTF.variable(kernel_val)
 
                 zth = KTH.eval(KTH.conv2d(xth, kernel_th, dim_ordering='th'))
@@ -572,7 +573,7 @@ class TestBackend(object):
 
                 kernel_val = np.random.random(kernel_shape) - 0.5
 
-                kernel_th = KTH.variable(convert_kernel(kernel_val))
+                kernel_th = KTH.variable(convert_kernel(kernel_val, dim_ordering='th'))
                 kernel_tf = KTF.variable(kernel_val)
 
                 zth = KTH.eval(KTH.conv3d(xth, kernel_th, dim_ordering='th'))
@@ -880,6 +881,35 @@ class TestBackend(object):
             assert k_s_d.shape == k_d.shape
             assert_allclose(k_s_d, k_d, atol=1e-05)
 
+    def test_map(self):
+        x = np.random.rand(10, 3).astype(np.float32)
+        for K in [KTF, KTH]:
+            kx = K.eval(K.map_fn(K.sum, x))
+
+            assert (10,) == kx.shape
+            assert_allclose(x.sum(axis=1), kx, atol=1e-05)
+
+    def test_foldl(self):
+        x = np.random.rand(10, 3).astype(np.float32)
+        for K in [KTF, KTH]:
+            kx = K.eval(K.foldl(lambda a, b: a+b, x))
+
+            assert (3,) == kx.shape
+            assert_allclose(x.sum(axis=0), kx, atol=1e-05)
+
+    def test_foldr(self):
+        # This test aims to make sure that we walk the array from right to left
+        # and checks it in the following way: multiplying left to right 1e-40
+        # cannot be held into a float32 so it causes an underflow while from
+        # right to left we have no such problem and the result is larger
+        x = np.array([1e-20, 1e-20, 10, 10, 10], dtype=np.float32)
+        for K in [KTF, KTH]:
+            p1 = K.eval(K.foldl(lambda a, b: a*b, x))
+            p2 = K.eval(K.foldr(lambda a, b: a*b, x))
+
+            assert p1 < p2
+            assert 9e-38 < p2 <= 1e-37
+
 
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/engine/test_training.py

@@ -4,10 +4,11 @@ from numpy.testing import assert_allclose
 
 from keras.layers import Dense, Dropout
 from keras.engine.topology import merge, Input
-from keras.engine.training import Model
-from keras.models import Sequential, Graph
+from keras.engine.training import Model, check_loss_and_target_compatibility
+from keras.models import Sequential
 from keras import backend as K
 from keras.utils.test_utils import keras_test
+from keras.callbacks import LambdaCallback
 
 
 @keras_test
@@ -146,17 +147,48 @@ def test_model_methods():
                               [output_a_np, output_b_np])
     assert len(out) == 4
 
+    # test starting from non-zero initial epoch
+    trained_epochs = []
+
+    def on_epoch_begin(epoch, logs):
+        trained_epochs.append(epoch)
+    tracker_cb = LambdaCallback(on_epoch_begin=on_epoch_begin)
+    out = model.fit([input_a_np, input_b_np],
+                    [output_a_np, output_b_np], nb_epoch=5, batch_size=4,
+                    initial_epoch=2, callbacks=[tracker_cb])
+    assert trained_epochs == [2, 3, 4]
+
+    # test starting from non-zero initial epoch for generator too
+    trained_epochs = []
+
+    def gen_data(batch_sz):
+        while True:
+            yield ([np.random.random((batch_sz, 3)), np.random.random((batch_sz, 3))],
+                   [np.random.random((batch_sz, 4)), np.random.random((batch_sz, 3))])
+    out = model.fit_generator(gen_data(4), samples_per_epoch=10, nb_epoch=5,
+                              initial_epoch=2, callbacks=[tracker_cb])
+    assert trained_epochs == [2, 3, 4]
+
     # test with a custom metric function
     mse = lambda y_true, y_pred: K.mean(K.pow(y_true - y_pred, 2))
-    model.compile(optimizer, loss, metrics=[mse],
+
+    def mse_powers(y_true, y_pred):
+        m = mse(y_true, y_pred)
+        return {
+            'mse_squared': K.pow(m, 2),
+            'mse_cubed': K.pow(m, 3)
+        }
+
+    model.compile(optimizer, loss, metrics=[mse, mse_powers],
                   sample_weight_mode=None)
 
     out = model.train_on_batch([input_a_np, input_b_np],
                                [output_a_np, output_b_np])
-    assert len(out) == 5
+    out_len = 1 + 2 * 4  # total loss, per layer: loss + 3 metrics
+    assert len(out) == out_len
     out = model.test_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np])
-    assert len(out) == 5
+    assert len(out) == out_len
 
     input_a_np = np.random.random((10, 3))
     input_b_np = np.random.random((10, 3))
@@ -193,5 +225,30 @@ def test_trainable_argument():
     assert_allclose(out, out_2)
 
 
+@keras_test
+def test_check_not_failing():
+    a = np.random.random((2, 1, 3))
+    check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [a.shape])
+    check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [(2, None, 3)])
+
+
+@keras_test
+def test_check_last_is_one():
+    a = np.random.random((2, 3, 1))
+    with pytest.raises(Exception) as exc:
+        check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [a.shape])
+
+    assert "You are passing a target array" in str(exc)
+
+
+@keras_test
+def test_check_bad_shape():
+    a = np.random.random((2, 3, 5))
+    with pytest.raises(Exception) as exc:
+        check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [(2, 3, 6)])
+
+    assert "targets to have the same shape" in str(exc)
+
+
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/layers/test_convolutional.py

@@ -8,6 +8,13 @@ from keras import backend as K
 from keras.layers import convolutional, pooling
 
 
+# TensorFlow does not support full convolution.
+if K._BACKEND == 'theano':
+    _convolution_border_modes = ['valid', 'same', 'full']
+else:
+    _convolution_border_modes = ['valid', 'same']
+
+
 @keras_test
 def test_convolution_1d():
     nb_samples = 2
@@ -16,7 +23,7 @@ def test_convolution_1d():
     filter_length = 3
     nb_filter = 3
 
-    for border_mode in ['valid', 'same']:
+    for border_mode in _convolution_border_modes:
         for subsample_length in [1, 2]:
             if border_mode == 'same' and subsample_length != 1:
                 continue
@@ -47,7 +54,7 @@ def test_atrous_conv_1d():
     filter_length = 3
     nb_filter = 3
 
-    for border_mode in ['valid', 'same']:
+    for border_mode in _convolution_border_modes:
         for subsample_length in [1, 2]:
             for atrous_rate in [1, 2]:
                 if border_mode == 'same' and subsample_length != 1:
@@ -101,7 +108,7 @@ def test_convolution_2d():
     nb_row = 10
     nb_col = 6
 
-    for border_mode in ['valid', 'same']:
+    for border_mode in _convolution_border_modes:
         for subsample in [(1, 1), (2, 2)]:
             if border_mode == 'same' and subsample != (1, 1):
                 continue
@@ -134,7 +141,7 @@ def test_deconvolution_2d():
     nb_row = 10
     nb_col = 6
 
-    for border_mode in ['valid', 'same']:
+    for border_mode in _convolution_border_modes:
         for subsample in [(1, 1), (2, 2)]:
             if border_mode == 'same' and subsample != (1, 1):
                 continue
@@ -175,7 +182,7 @@ def test_atrous_conv_2d():
     nb_row = 10
     nb_col = 6
 
-    for border_mode in ['valid', 'same']:
+    for border_mode in _convolution_border_modes:
         for subsample in [(1, 1), (2, 2)]:
             for atrous_rate in [(1, 1), (2, 2)]:
                 if border_mode == 'same' and subsample != (1, 1):
@@ -214,7 +221,7 @@ def test_separable_conv_2d():
     nb_row = 10
     nb_col = 6
 
-    for border_mode in ['valid', 'same']:
+    for border_mode in _convolution_border_modes:
         for subsample in [(1, 1), (2, 2)]:
             for multiplier in [1, 2]:
                 if border_mode == 'same' and subsample != (1, 1):
@@ -269,6 +276,22 @@ def test_globalpooling_2d():
                input_shape=(3, 5, 6, 4))
 
 
+@keras_test
+def test_globalpooling_3d():
+    layer_test(pooling.GlobalMaxPooling3D,
+               kwargs={'dim_ordering': 'th'},
+               input_shape=(3, 4, 3, 4, 3))
+    layer_test(pooling.GlobalMaxPooling3D,
+               kwargs={'dim_ordering': 'tf'},
+               input_shape=(3, 4, 3, 4, 3))
+    layer_test(pooling.GlobalAveragePooling3D,
+               kwargs={'dim_ordering': 'th'},
+               input_shape=(3, 4, 3, 4, 3))
+    layer_test(pooling.GlobalAveragePooling3D,
+               kwargs={'dim_ordering': 'tf'},
+               input_shape=(3, 4, 3, 4, 3))
+
+
 @keras_test
 def test_maxpooling_2d():
     pool_size = (3, 3)
@@ -283,12 +306,10 @@ def test_maxpooling_2d():
 
 @keras_test
 def test_averagepooling_2d():
-    pool_size = (3, 3)
-
     for border_mode in ['valid', 'same']:
         for pool_size in [(2, 2), (3, 3), (4, 4), (5, 5)]:
             for strides in [(1, 1), (2, 2)]:
-                layer_test(convolutional.MaxPooling2D,
+                layer_test(convolutional.AveragePooling2D,
                            kwargs={'strides': strides,
                                    'border_mode': border_mode,
                                    'pool_size': pool_size},
@@ -308,7 +329,7 @@ def test_convolution_3d():
     input_len_dim2 = 11
     input_len_dim3 = 12
 
-    for border_mode in ['same', 'valid']:
+    for border_mode in _convolution_border_modes:
         for subsample in [(1, 1, 1), (2, 2, 2)]:
             if border_mode == 'same' and subsample != (1, 1, 1):
                 continue
@@ -363,38 +384,120 @@ def test_averagepooling_3d():
                    input_shape=(3, 4, 11, 12, 10))
 
 
+@keras_test
+def test_zero_padding_1d():
+    nb_samples = 2
+    input_dim = 2
+    nb_steps = 5
+    shape = (nb_samples, nb_steps, input_dim)
+    input = np.ones(shape)
+
+    # basic test
+    layer_test(convolutional.ZeroPadding1D,
+               kwargs={'padding': 2},
+               input_shape=input.shape)
+    layer_test(convolutional.ZeroPadding1D,
+               kwargs={'padding': (1, 2)},
+               input_shape=input.shape)
+    layer_test(convolutional.ZeroPadding1D,
+               kwargs={'padding': {'left_pad': 1, 'right_pad': 2}},
+               input_shape=input.shape)
+
+    # correctness test
+    layer = convolutional.ZeroPadding1D(padding=2)
+    layer.build(shape)
+    output = layer(K.variable(input))
+    np_output = K.eval(output)
+    for offset in [0, 1, -1, -2]:
+        assert_allclose(np_output[:, offset, :], 0.)
+    assert_allclose(np_output[:, 2:-2, :], 1.)
+
+    layer = convolutional.ZeroPadding1D(padding=(1, 2))
+    layer.build(shape)
+    output = layer(K.variable(input))
+    np_output = K.eval(output)
+    for left_offset in [0]:
+        assert_allclose(np_output[:, left_offset, :], 0.)
+    for right_offset in [-1, -2]:
+        assert_allclose(np_output[:, right_offset, :], 0.)
+    assert_allclose(np_output[:, 1:-2, :], 1.)
+    layer.get_config()
+
+
 @keras_test
 def test_zero_padding_2d():
     nb_samples = 2
     stack_size = 2
-    input_nb_row = 11
-    input_nb_col = 12
+    input_nb_row = 4
+    input_nb_col = 5
+    dim_ordering = K.image_dim_ordering()
+    assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
 
-    input = np.ones((nb_samples, input_nb_row, input_nb_col, stack_size))
+    if dim_ordering == 'tf':
+        input = np.ones((nb_samples, input_nb_row, input_nb_col, stack_size))
+    elif dim_ordering == 'th':
+        input = np.ones((nb_samples, stack_size, input_nb_row, input_nb_col))
 
     # basic test
     layer_test(convolutional.ZeroPadding2D,
                kwargs={'padding': (2, 2)},
                input_shape=input.shape)
+    layer_test(convolutional.ZeroPadding2D,
+               kwargs={'padding': (1, 2, 3, 4)},
+               input_shape=input.shape)
+    layer_test(convolutional.ZeroPadding2D,
+               kwargs={'padding': {'top_pad': 1, 'bottom_pad': 2, 'left_pad': 3, 'right_pad': 4}},
+               input_shape=input.shape)
 
     # correctness test
     layer = convolutional.ZeroPadding2D(padding=(2, 2))
-    layer.set_input(K.variable(input), shape=input.shape)
+    layer.build(input.shape)
+    output = layer(K.variable(input))
+    np_output = K.eval(output)
+    if dim_ordering == 'tf':
+        for offset in [0, 1, -1, -2]:
+            assert_allclose(np_output[:, offset, :, :], 0.)
+            assert_allclose(np_output[:, :, offset, :], 0.)
+        assert_allclose(np_output[:, 2:-2, 2:-2, :], 1.)
+    elif dim_ordering == 'th':
+        for offset in [0, 1, -1, -2]:
+            assert_allclose(np_output[:, :, offset, :], 0.)
+            assert_allclose(np_output[:, :, :, offset], 0.)
+        assert_allclose(np_output[:, 2:-2, 2:-2, :], 1.)
 
-    out = K.eval(layer.output)
-    for offset in [0, 1, -1, -2]:
-        assert_allclose(out[:, offset, :, :], 0.)
-        assert_allclose(out[:, :, offset, :], 0.)
-    assert_allclose(out[:, 2:-2, 2:-2, :], 1.)
+    layer = convolutional.ZeroPadding2D(padding=(1, 2, 3, 4))
+    layer.build(input.shape)
+    output = layer(K.variable(input))
+    np_output = K.eval(output)
+    if dim_ordering == 'tf':
+        for top_offset in [0]:
+            assert_allclose(np_output[:, top_offset, :, :], 0.)
+        for bottom_offset in [-1, -2]:
+            assert_allclose(np_output[:, bottom_offset, :, :], 0.)
+        for left_offset in [0, 1, 2]:
+            assert_allclose(np_output[:, :, left_offset, :], 0.)
+        for right_offset in [-1, -2, -3, -4]:
+            assert_allclose(np_output[:, :, right_offset, :], 0.)
+        assert_allclose(np_output[:, 1:-2, 3:-4, :], 1.)
+    elif dim_ordering == 'th':
+        for top_offset in [0]:
+            assert_allclose(np_output[:, :, top_offset, :], 0.)
+        for bottom_offset in [-1, -2]:
+            assert_allclose(np_output[:, :, bottom_offset, :], 0.)
+        for left_offset in [0, 1, 2]:
+            assert_allclose(np_output[:, :, :, left_offset], 0.)
+        for right_offset in [-1, -2, -3, -4]:
+            assert_allclose(np_output[:, :, :, right_offset], 0.)
+        assert_allclose(np_output[:, :, 1:-2, 3:-4], 1.)
     layer.get_config()
 
 
 def test_zero_padding_3d():
     nb_samples = 2
     stack_size = 2
-    input_len_dim1 = 10
-    input_len_dim2 = 11
-    input_len_dim3 = 12
+    input_len_dim1 = 4
+    input_len_dim2 = 5
+    input_len_dim3 = 3
 
     input = np.ones((nb_samples,
                      input_len_dim1, input_len_dim2, input_len_dim3,
@@ -407,13 +510,14 @@ def test_zero_padding_3d():
 
     # correctness test
     layer = convolutional.ZeroPadding3D(padding=(2, 2, 2))
-    layer.set_input(K.variable(input), shape=input.shape)
-    out = K.eval(layer.output)
+    layer.build(input.shape)
+    output = layer(K.variable(input))
+    np_output = K.eval(output)
     for offset in [0, 1, -1, -2]:
-        assert_allclose(out[:, offset, :, :, :], 0.)
-        assert_allclose(out[:, :, offset, :, :], 0.)
-        assert_allclose(out[:, :, :, offset, :], 0.)
-    assert_allclose(out[:, 2:-2, 2:-2, 2:-2, :], 1.)
+        assert_allclose(np_output[:, offset, :, :, :], 0.)
+        assert_allclose(np_output[:, :, offset, :, :], 0.)
+        assert_allclose(np_output[:, :, :, offset, :], 0.)
+    assert_allclose(np_output[:, 2:-2, 2:-2, 2:-2, :], 1.)
     layer.get_config()
 
 
@@ -444,15 +548,15 @@ def test_upsampling_2d():
                 layer = convolutional.UpSampling2D(
                     size=(length_row, length_col),
                     dim_ordering=dim_ordering)
-                layer.set_input(K.variable(input), shape=input.shape)
-
-                out = K.eval(layer.output)
+                layer.build(input.shape)
+                output = layer(K.variable(input))
+                np_output = K.eval(output)
                 if dim_ordering == 'th':
-                    assert out.shape[2] == length_row * input_nb_row
-                    assert out.shape[3] == length_col * input_nb_col
+                    assert np_output.shape[2] == length_row * input_nb_row
+                    assert np_output.shape[3] == length_col * input_nb_col
                 else:  # tf
-                    assert out.shape[1] == length_row * input_nb_row
-                    assert out.shape[2] == length_col * input_nb_col
+                    assert np_output.shape[1] == length_row * input_nb_row
+                    assert np_output.shape[2] == length_col * input_nb_col
 
                 # compare with numpy
                 if dim_ordering == 'th':
@@ -462,7 +566,7 @@ def test_upsampling_2d():
                     expected_out = np.repeat(input, length_row, axis=1)
                     expected_out = np.repeat(expected_out, length_col, axis=2)
 
-                assert_allclose(out, expected_out)
+                assert_allclose(np_output, expected_out)
 
 
 def test_upsampling_3d():
@@ -485,17 +589,17 @@ def test_upsampling_3d():
                     layer = convolutional.UpSampling3D(
                         size=(length_dim1, length_dim2, length_dim3),
                         dim_ordering=dim_ordering)
-                    layer.set_input(K.variable(input), shape=input.shape)
-
-                    out = K.eval(layer.output)
+                    layer.build(input.shape)
+                    output = layer(K.variable(input))
+                    np_output = K.eval(output)
                     if dim_ordering == 'th':
-                        assert out.shape[2] == length_dim1 * input_len_dim1
-                        assert out.shape[3] == length_dim2 * input_len_dim2
-                        assert out.shape[4] == length_dim3 * input_len_dim3
+                        assert np_output.shape[2] == length_dim1 * input_len_dim1
+                        assert np_output.shape[3] == length_dim2 * input_len_dim2
+                        assert np_output.shape[4] == length_dim3 * input_len_dim3
                     else:  # tf
-                        assert out.shape[1] == length_dim1 * input_len_dim1
-                        assert out.shape[2] == length_dim2 * input_len_dim2
-                        assert out.shape[3] == length_dim3 * input_len_dim3
+                        assert np_output.shape[1] == length_dim1 * input_len_dim1
+                        assert np_output.shape[2] == length_dim2 * input_len_dim2
+                        assert np_output.shape[3] == length_dim3 * input_len_dim3
 
                     # compare with numpy
                     if dim_ordering == 'th':
@@ -507,13 +611,13 @@ def test_upsampling_3d():
                         expected_out = np.repeat(expected_out, length_dim2, axis=2)
                         expected_out = np.repeat(expected_out, length_dim3, axis=3)
 
-                    assert_allclose(out, expected_out)
+                    assert_allclose(np_output, expected_out)
 
 
 @keras_test
 def test_cropping_1d():
     nb_samples = 2
-    time_length = 10
+    time_length = 4
     input_len_dim1 = 2
     input = np.random.rand(nb_samples, time_length, input_len_dim1)
 
@@ -531,32 +635,35 @@ def test_cropping_2d():
     dim_ordering = K.image_dim_ordering()
 
     if dim_ordering == 'th':
-        input = np.random.rand(nb_samples, stack_size, input_len_dim1, input_len_dim2)
+        input = np.random.rand(nb_samples, stack_size,
+                               input_len_dim1, input_len_dim2)
     else:
-        input = np.random.rand(nb_samples, input_len_dim1, input_len_dim2, stack_size)
+        input = np.random.rand(nb_samples,
+                               input_len_dim1, input_len_dim2,
+                               stack_size)
     # basic test
     layer_test(convolutional.Cropping2D,
                kwargs={'cropping': cropping,
                        'dim_ordering': dim_ordering},
                input_shape=input.shape)
     # correctness test
-    layer = convolutional.Cropping2D(cropping=cropping, dim_ordering=dim_ordering)
-    layer.set_input(K.variable(input), shape=input.shape)
-
-    out = K.eval(layer.output)
+    layer = convolutional.Cropping2D(cropping=cropping,
+                                     dim_ordering=dim_ordering)
+    layer.build(input.shape)
+    output = layer(K.variable(input))
+    np_output = K.eval(output)
     # compare with numpy
     if dim_ordering == 'th':
         expected_out = input[:,
                              :,
-                             cropping[0][0]:-cropping[0][1],
-                             cropping[1][0]:-cropping[1][1]]
+                             cropping[0][0]: -cropping[0][1],
+                             cropping[1][0]: -cropping[1][1]]
     else:
         expected_out = input[:,
-                             cropping[0][0]:-cropping[0][1],
-                             cropping[1][0]:-cropping[1][1],
+                             cropping[0][0]: -cropping[0][1],
+                             cropping[1][0]: -cropping[1][1],
                              :]
-
-    assert_allclose(out, expected_out)
+    assert_allclose(np_output, expected_out)
 
 
 def test_cropping_3d():
@@ -569,34 +676,37 @@ def test_cropping_3d():
     dim_ordering = K.image_dim_ordering()
 
     if dim_ordering == 'th':
-        input = np.random.rand(nb_samples, stack_size, input_len_dim1, input_len_dim2, input_len_dim3)
+        input = np.random.rand(nb_samples, stack_size,
+                               input_len_dim1, input_len_dim2, input_len_dim3)
     else:
-        input = np.random.rand(nb_samples, input_len_dim1, input_len_dim2, input_len_dim3, stack_size)
+        input = np.random.rand(nb_samples,
+                               input_len_dim1, input_len_dim2,
+                               input_len_dim3, stack_size)
     # basic test
     layer_test(convolutional.Cropping3D,
                kwargs={'cropping': cropping,
                        'dim_ordering': dim_ordering},
                input_shape=input.shape)
     # correctness test
-    layer = convolutional.Cropping3D(cropping=cropping, dim_ordering=dim_ordering)
-    layer.set_input(K.variable(input), shape=input.shape)
-
-    out = K.eval(layer.output)
+    layer = convolutional.Cropping3D(cropping=cropping,
+                                     dim_ordering=dim_ordering)
+    layer.build(input.shape)
+    output = layer(K.variable(input))
+    np_output = K.eval(output)
     # compare with numpy
     if dim_ordering == 'th':
         expected_out = input[:,
                              :,
-                             cropping[0][0]:-cropping[0][1],
-                             cropping[1][0]:-cropping[1][1],
-                             cropping[2][0]:-cropping[2][1]]
+                             cropping[0][0]: -cropping[0][1],
+                             cropping[1][0]: -cropping[1][1],
+                             cropping[2][0]: -cropping[2][1]]
     else:
         expected_out = input[:,
-                             cropping[0][0]:-cropping[0][1],
-                             cropping[1][0]:-cropping[1][1],
-                             cropping[2][0]:-cropping[2][1],
+                             cropping[0][0]: -cropping[0][1],
+                             cropping[1][0]: -cropping[1][1],
+                             cropping[2][0]: -cropping[2][1],
                              :]
-
-    assert_allclose(out, expected_out)
+    assert_allclose(np_output, expected_out)
 
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/layers/test_convolutional_recurrent.py

@@ -0,0 +1,130 @@
+import pytest
+import numpy as np
+from numpy.testing import assert_allclose
+
+from keras import backend as K
+from keras.models import Sequential
+from keras.layers import convolutional_recurrent
+from keras.utils.test_utils import layer_test
+from keras import regularizers
+
+
+def test_recurrent_convolutional():
+    nb_row = 3
+    nb_col = 3
+    nb_filter = 5
+    nb_samples = 2
+    input_channel = 2
+    input_nb_row = 5
+    input_nb_col = 5
+    sequence_len = 2
+    for dim_ordering in ['th', 'tf']:
+
+        if dim_ordering == 'th':
+            input = np.random.rand(nb_samples, sequence_len,
+                                   input_channel,
+                                   input_nb_row, input_nb_col)
+        else:  # tf
+            input = np.random.rand(nb_samples, sequence_len,
+                                   input_nb_row, input_nb_col,
+                                   input_channel)
+
+        for return_sequences in [True, False]:
+            # test for ouptput shape:
+            output = layer_test(convolutional_recurrent.ConvLSTM2D,
+                                kwargs={'dim_ordering': dim_ordering,
+                                        'return_sequences': return_sequences,
+                                        'nb_filter': nb_filter,
+                                        'nb_row': nb_row,
+                                        'nb_col': nb_col,
+                                        'border_mode': "same"},
+                                input_shape=input.shape)
+
+            output_shape = [nb_samples, input_nb_row, input_nb_col]
+
+            if dim_ordering == 'th':
+                output_shape.insert(1, nb_filter)
+            else:
+                output_shape.insert(3, nb_filter)
+
+            if return_sequences:
+                output_shape.insert(1, sequence_len)
+
+            assert output.shape == tuple(output_shape)
+
+            # No need to check statefulness for both
+            if dim_ordering == 'th' or return_sequences:
+                continue
+
+            # Tests for statefulness
+            model = Sequential()
+            kwargs = {'dim_ordering': dim_ordering,
+                      'return_sequences': return_sequences,
+                      'nb_filter': nb_filter,
+                      'nb_row': nb_row,
+                      'nb_col': nb_col,
+                      'stateful': True,
+                      'batch_input_shape': input.shape,
+                      'border_mode': "same"}
+            layer = convolutional_recurrent.ConvLSTM2D(**kwargs)
+
+            model.add(layer)
+            model.compile(optimizer='sgd', loss='mse')
+            out1 = model.predict(np.ones_like(input))
+            assert(out1.shape == tuple(output_shape))
+
+            # train once so that the states change
+            model.train_on_batch(np.ones_like(input),
+                                 np.ones_like(output))
+            out2 = model.predict(np.ones_like(input))
+
+            # if the state is not reset, output should be different
+            assert(out1.max() != out2.max())
+
+            # check that output changes after states are reset
+            # (even though the model itself didn't change)
+            layer.reset_states()
+            out3 = model.predict(np.ones_like(input))
+            assert(out2.max() != out3.max())
+
+            # check that container-level reset_states() works
+            model.reset_states()
+            out4 = model.predict(np.ones_like(input))
+            assert_allclose(out3, out4, atol=1e-5)
+
+            # check that the call to `predict` updated the states
+            out5 = model.predict(np.ones_like(input))
+            assert(out4.max() != out5.max())
+
+            # check regularizers
+            kwargs = {'dim_ordering': dim_ordering,
+                      'return_sequences': return_sequences,
+                      'nb_filter': nb_filter,
+                      'nb_row': nb_row,
+                      'nb_col': nb_col,
+                      'stateful': True,
+                      'batch_input_shape': input.shape,
+                      'W_regularizer': regularizers.WeightRegularizer(l1=0.01),
+                      'U_regularizer': regularizers.WeightRegularizer(l1=0.01),
+                      'b_regularizer': 'l2',
+                      'border_mode': "same"}
+
+            layer = convolutional_recurrent.ConvLSTM2D(**kwargs)
+            layer.build(input.shape)
+            output = layer(K.variable(np.ones(input.shape)))
+            K.eval(output)
+
+            # check dropout
+            layer_test(convolutional_recurrent.ConvLSTM2D,
+                       kwargs={'dim_ordering': dim_ordering,
+                               'return_sequences': return_sequences,
+                               'nb_filter': nb_filter,
+                               'nb_row': nb_row,
+                               'nb_col': nb_col,
+                               'border_mode': "same",
+                               'dropout_W': 0.1,
+                               'dropout_U': 0.1},
+                       input_shape=input.shape)
+
+if __name__ == '__main__':
+    pytest.main([__file__])

tests/keras/layers/test_core.py

@@ -15,7 +15,7 @@ def test_masking():
 
 @keras_test
 def test_merge():
-    from keras.layers import Input, merge, Merge
+    from keras.layers import Input, merge, Merge, Masking
     from keras.models import Model
 
     # test modes: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot'.
@@ -53,7 +53,8 @@ def test_merge():
     input_b = Input(shape=input_shapes[1][1:])
     merged = merge([input_a, input_b],
                    mode=lambda tup: K.concatenate([tup[0], tup[1]]),
-                   output_shape=lambda tup: (tup[0][:-1],) + (tup[0][-1] + tup[1][-1],))
+                   output_shape=lambda tup: tup[0][:-1] + (tup[0][-1] + tup[1][-1],))
+    model = Model([input_a, input_b], merged)
     expected_output_shape = model.get_output_shape_for(input_shapes)
     actual_output_shape = model.predict(inputs).shape
     assert expected_output_shape == actual_output_shape
@@ -65,17 +66,18 @@ def test_merge():
     # test function with output_shape function
     def fn_mode(tup):
         x, y = tup
-        return K.concatenate([x, y])
+        return K.concatenate([x, y], axis=1)
 
     def fn_output_shape(tup):
         s1, s2 = tup
-        return (s1[:-1],) + (s1[-1] + s2[-1],)
+        return (s1[0], s1[1] + s2[1]) + s1[2:]
 
     input_a = Input(shape=input_shapes[0][1:])
     input_b = Input(shape=input_shapes[1][1:])
     merged = merge([input_a, input_b],
                    mode=fn_mode,
                    output_shape=fn_output_shape)
+    model = Model([input_a, input_b], merged)
     expected_output_shape = model.get_output_shape_for(input_shapes)
     actual_output_shape = model.predict(inputs).shape
     assert expected_output_shape == actual_output_shape
@@ -84,6 +86,74 @@ def test_merge():
     model = Model.from_config(config)
     model.compile('rmsprop', 'mse')
 
+    # test function with output_mask function
+    # time dimension is required for masking
+    input_shapes = [(4, 3, 2), (4, 3, 2)]
+    inputs = [np.random.random(shape) for shape in input_shapes]
+
+    def fn_output_mask(tup):
+        x_mask, y_mask = tup
+        return K.concatenate([x_mask, y_mask])
+
+    input_a = Input(shape=input_shapes[0][1:])
+    input_b = Input(shape=input_shapes[1][1:])
+    a = Masking()(input_a)
+    b = Masking()(input_b)
+    merged = merge([a, b], mode=fn_mode, output_shape=fn_output_shape, output_mask=fn_output_mask)
+    model = Model([input_a, input_b], merged)
+    expected_output_shape = model.get_output_shape_for(input_shapes)
+    actual_output_shape = model.predict(inputs).shape
+    assert expected_output_shape == actual_output_shape
+
+    config = model.get_config()
+    model = Model.from_config(config)
+    model.compile('rmsprop', 'mse')
+
+    mask_inputs = (np.zeros(input_shapes[0][:-1]), np.ones(input_shapes[1][:-1]))
+    expected_mask_output = np.concatenate(mask_inputs, axis=-1)
+    mask_input_placeholders = [K.placeholder(shape=input_shape[:-1]) for input_shape in input_shapes]
+    mask_output = model.layers[-1]._output_mask(mask_input_placeholders)
+    assert np.all(K.function(mask_input_placeholders, [mask_output])(mask_inputs)[0] == expected_mask_output)
+
+    # test lambda with output_mask lambda
+    input_a = Input(shape=input_shapes[0][1:])
+    input_b = Input(shape=input_shapes[1][1:])
+    a = Masking()(input_a)
+    b = Masking()(input_b)
+    merged = merge([a, b], mode=lambda tup: K.concatenate([tup[0], tup[1]], axis=1),
+                   output_shape=lambda tup: (tup[0][0], tup[0][1] + tup[1][1]) + tup[0][2:],
+                   output_mask=lambda tup: K.concatenate([tup[0], tup[1]]))
+    model = Model([input_a, input_b], merged)
+    expected_output_shape = model.get_output_shape_for(input_shapes)
+    actual_output_shape = model.predict(inputs).shape
+    assert expected_output_shape == actual_output_shape
+
+    config = model.get_config()
+    model = Model.from_config(config)
+    model.compile('rmsprop', 'mse')
+
+    mask_output = model.layers[-1]._output_mask(mask_input_placeholders)
+    assert np.all(K.function(mask_input_placeholders, [mask_output])(mask_inputs)[0] == expected_mask_output)
+
+    # test with arguments
+    input_shapes = [(3, 2), (3, 2)]
+    inputs = [np.random.random(shape) for shape in input_shapes]
+
+    def fn_mode(tup, a, b):
+        x, y = tup
+        return x * a + y * b
+
+    input_a = Input(shape=input_shapes[0][1:])
+    input_b = Input(shape=input_shapes[1][1:])
+    merged = merge([input_a, input_b], mode=fn_mode, output_shape=lambda s: s[0], arguments={'a': 0.7, 'b': 0.3})
+    model = Model([input_a, input_b], merged)
+    output = model.predict(inputs)
+
+    config = model.get_config()
+    model = Model.from_config(config)
+
+    assert np.all(model.predict(inputs) == output)
+
 
 @keras_test
 def test_merge_mask_2d():
@@ -153,6 +223,10 @@ def test_dropout():
                kwargs={'p': 0.5},
                input_shape=(3, 2))
 
+    layer_test(core.SpatialDropout1D,
+               kwargs={'p': 0.5},
+               input_shape=(2, 3, 4))
+
     layer_test(core.SpatialDropout2D,
                kwargs={'p': 0.5},
                input_shape=(2, 3, 4, 5))
@@ -212,6 +286,11 @@ def test_lambda():
                kwargs={'function': lambda x: x + 1},
                input_shape=(3, 2))
 
+    layer_test(Lambda,
+               kwargs={'function': lambda x, a, b: x * a + b,
+                       'arguments': {'a': 0.6, 'b': 0.4}},
+               input_shape=(3, 2))
+
     # test serialization with function
     def f(x):
         return x + 1

tests/keras/layers/test_normalization.py

@@ -2,10 +2,10 @@ import pytest
 import numpy as np
 from numpy.testing import assert_allclose
 
-from keras.layers.core import Dense, Activation
+from keras.layers import Dense, Activation, Input
 from keras.utils.test_utils import layer_test, keras_test
 from keras.layers import normalization
-from keras.models import Sequential, Graph
+from keras.models import Sequential, Model
 from keras import backend as K
 
 input_1 = np.arange(10)
@@ -78,5 +78,33 @@ def test_batchnorm_mode_1():
             assert_allclose(K.eval(K.std(out)), 0.0, atol=1e-1)
 
 
+@keras_test
+def test_shared_batchnorm():
+    '''Test that a BN layer can be shared
+    across different data streams.
+    '''
+    # Test single layer reuse
+    bn = normalization.BatchNormalization(input_shape=(10,), mode=0)
+    x1 = Input(shape=(10,))
+    bn(x1)
+
+    x2 = Input(shape=(10,))
+    y2 = bn(x2)
+
+    x = np.random.normal(loc=5.0, scale=10.0, size=(2, 10))
+    model = Model(x2, y2)
+    assert len(model.updates) == 2
+    model.compile('sgd', 'mse')
+    model.train_on_batch(x, x)
+
+    # Test model-level reuse
+    x3 = Input(shape=(10,))
+    y3 = model(x3)
+    new_model = Model(x3, y3)
+    assert len(model.updates) == 2
+    new_model.compile('sgd', 'mse')
+    new_model.train_on_batch(x, x)
+
+
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/layers/test_recurrent.py

@@ -15,18 +15,29 @@ nb_samples, timesteps, embedding_dim, output_dim = 2, 5, 4, 3
 embedding_num = 12
 
 
-def _runner(layer_class):
+def rnn_test(f):
     """
     All the recurrent layers share the same interface,
     so we can run through them with a single function.
     """
-    # check return_sequences
+    f = keras_test(f)
+    return pytest.mark.parametrize("layer_class", [
+        recurrent.SimpleRNN,
+        recurrent.GRU,
+        recurrent.LSTM
+    ])(f)
+
+
+@rnn_test
+def test_return_sequences(layer_class):
     layer_test(layer_class,
                kwargs={'output_dim': output_dim,
                        'return_sequences': True},
                input_shape=(nb_samples, timesteps, embedding_dim))
 
-    # check dynamic behavior
+
+@rnn_test
+def test_dynamic_behavior(layer_class):
     layer = layer_class(output_dim, input_dim=embedding_dim)
     model = Sequential()
     model.add(layer)
@@ -35,21 +46,27 @@ def _runner(layer_class):
     y = np.random.random((nb_samples, output_dim))
     model.train_on_batch(x, y)
 
-    # check dropout
+
+@rnn_test
+def test_dropout(layer_class):
     layer_test(layer_class,
                kwargs={'output_dim': output_dim,
                        'dropout_U': 0.1,
                        'dropout_W': 0.1},
                input_shape=(nb_samples, timesteps, embedding_dim))
 
-    # check implementation modes
+
+@rnn_test
+def test_implementation_mode(layer_class):
     for mode in ['cpu', 'mem', 'gpu']:
         layer_test(layer_class,
                    kwargs={'output_dim': output_dim,
                            'consume_less': mode},
                    input_shape=(nb_samples, timesteps, embedding_dim))
 
-    # check statefulness
+
+@rnn_test
+def test_statefulness(layer_class):
     model = Sequential()
     model.add(embeddings.Embedding(embedding_num, embedding_dim,
                                    mask_zero=True,
@@ -103,31 +120,18 @@ def _runner(layer_class):
 
     assert_allclose(out7, out6, atol=1e-5)
 
-    # check regularizers
+
+@rnn_test
+def test_regularizer(layer_class):
     layer = layer_class(output_dim, return_sequences=False, weights=None,
                         batch_input_shape=(nb_samples, timesteps, embedding_dim),
                         W_regularizer=regularizers.WeightRegularizer(l1=0.01),
                         U_regularizer=regularizers.WeightRegularizer(l1=0.01),
                         b_regularizer='l2')
     shape = (nb_samples, timesteps, embedding_dim)
-    layer.set_input(K.variable(np.ones(shape)),
-                    shape=shape)
-    K.eval(layer.output)
-
-
-@keras_test
-def test_SimpleRNN():
-    _runner(recurrent.SimpleRNN)
-
-
-@keras_test
-def test_GRU():
-    _runner(recurrent.GRU)
-
-
-@keras_test
-def test_LSTM():
-    _runner(recurrent.LSTM)
+    layer.build(shape)
+    output = layer(K.variable(np.ones(shape)))
+    K.eval(output)
 
 
 @keras_test
@@ -136,15 +140,30 @@ def test_masking_layer():
     https://github.com/fchollet/keras/issues/1567
 
     '''
-    model = Sequential()
-    model.add(Masking(input_shape=(3, 4)))
-    model.add(recurrent.LSTM(output_dim=5, return_sequences=True))
-    model.compile(loss='categorical_crossentropy', optimizer='adam')
     I = np.random.random((6, 3, 4))
     V = np.abs(np.random.random((6, 3, 5)))
     V /= V.sum(axis=-1, keepdims=True)
+
+    model = Sequential()
+    model.add(Masking(input_shape=(3, 4)))
+    model.add(recurrent.LSTM(output_dim=5, return_sequences=True, unroll=False))
+    model.compile(loss='categorical_crossentropy', optimizer='adam')
     model.fit(I, V, nb_epoch=1, batch_size=100, verbose=1)
 
+    model = Sequential()
+    model.add(Masking(input_shape=(3, 4)))
+    model.add(recurrent.LSTM(output_dim=5, return_sequences=True, unroll=True))
+    model.compile(loss='categorical_crossentropy', optimizer='adam')
+    model.fit(I, V, nb_epoch=1, batch_size=100, verbose=1)
+
+
+@rnn_test
+def test_from_config(layer_class):
+    for stateful in (False, True):
+        l1 = layer_class(output_dim=1, stateful=stateful)
+        l2 = layer_class.from_config(l1.get_config())
+        assert l1.get_config() == l2.get_config()
+
 
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/layers/test_wrappers.py

@@ -115,6 +115,13 @@ def test_Bidirectional():
         model.compile(loss='mse', optimizer='sgd')
         model.fit(x, y, nb_epoch=1, batch_size=1)
 
+        # Bidirectional and stateful
+        input = Input(batch_shape=(1, timesteps, dim))
+        output = wrappers.Bidirectional(rnn(output_dim, stateful=True), merge_mode=mode)(input)
+        model = Model(input, output)
+        model.compile(loss='mse', optimizer='sgd')
+        model.fit(x, y, nb_epoch=1, batch_size=1)
+
 
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/test_activations.py

@@ -131,6 +131,23 @@ def test_relu():
     assert_allclose(result, test_values, rtol=1e-05)
 
 
+def test_elu():
+    x = K.placeholder(ndim=2)
+    f = K.function([x], [activations.elu(x, 0.5)])
+
+    test_values = get_standard_values()
+    result = f([test_values])[0]
+
+    # because no negatives in test values
+    assert_allclose(result, test_values, rtol=1e-05)
+
+    negative_values = np.array([[-1, -2]], dtype=K.floatx())
+    result = f([negative_values])[0]
+    true_result = (np.exp(negative_values) - 1) / 2
+
+    assert_allclose(result, true_result)
+
+
 def test_tanh():
     test_values = get_standard_values()
 

tests/keras/test_callbacks.py

@@ -1,11 +1,15 @@
-import pytest
 import os
 import sys
+import multiprocessing
+
 import numpy as np
+import pytest
+from keras import optimizers
+
 np.random.seed(1337)
 
 from keras import callbacks
-from keras.models import Graph, Sequential
+from keras.models import Sequential
 from keras.layers.core import Dense
 from keras.utils.test_utils import get_test_data
 from keras import backend as K
@@ -147,13 +151,47 @@ def test_LearningRateScheduler():
     assert (float(K.get_value(model.optimizer.lr)) - 0.2) < K.epsilon()
 
 
-@pytest.mark.skipif((K._BACKEND != 'tensorflow'),
+def test_ReduceLROnPlateau():
+    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
+                                                         nb_test=test_samples,
+                                                         input_shape=(input_dim,),
+                                                         classification=True,
+                                                         nb_class=nb_class)
+    y_test = np_utils.to_categorical(y_test)
+    y_train = np_utils.to_categorical(y_train)
+
+    def make_model():
+        np.random.seed(1337)
+        model = Sequential()
+        model.add(Dense(nb_hidden, input_dim=input_dim, activation='relu'))
+        model.add(Dense(nb_class, activation='softmax'))
+
+        model.compile(loss='categorical_crossentropy',
+                      optimizer=optimizers.SGD(lr=0.1),
+                      metrics=['accuracy'])
+        return model
+
+    model = make_model()
+
+    # This should reduce the LR after the first epoch (due to high epsilon).
+    cbks = [callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, epsilon=10, patience=1, cooldown=5)]
+    model.fit(X_train, y_train, batch_size=batch_size,
+              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=5, verbose=2)
+    assert np.allclose(float(K.get_value(model.optimizer.lr)), 0.01, atol=K.epsilon())
+
+    model = make_model()
+    cbks = [callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, epsilon=0, patience=1, cooldown=5)]
+    model.fit(X_train, y_train, batch_size=batch_size,
+              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=5, verbose=2)
+    assert np.allclose(float(K.get_value(model.optimizer.lr)), 0.1, atol=K.epsilon())
+
+
+@pytest.mark.skipif((K.backend() != 'tensorflow'),
                     reason="Requires tensorflow backend")
 def test_TensorBoard():
     import shutil
     import tensorflow as tf
     import keras.backend.tensorflow_backend as KTF
-    old_session = KTF.get_session()
     filepath = './logs'
     (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
                                                          nb_test=test_samples,
@@ -185,92 +223,112 @@ def test_TensorBoard():
                 yield {'X_vars': X_test, 'output': y_test}
 
     # case 1 Sequential
+    model = Sequential()
+    model.add(Dense(nb_hidden, input_dim=input_dim, activation='relu'))
+    model.add(Dense(nb_class, activation='softmax'))
+    model.compile(loss='categorical_crossentropy',
+                  optimizer='sgd',
+                  metrics=['accuracy'])
 
-    with tf.Graph().as_default():
-        session = tf.Session('')
-        KTF.set_session(session)
-        model = Sequential()
-        model.add(Dense(nb_hidden, input_dim=input_dim, activation='relu'))
-        model.add(Dense(nb_class, activation='softmax'))
-        model.compile(loss='categorical_crossentropy',
-                      optimizer='sgd',
-                      metrics=['accuracy'])
+    tsb = callbacks.TensorBoard(log_dir=filepath, histogram_freq=1)
+    cbks = [tsb]
 
-        tsb = callbacks.TensorBoard(log_dir=filepath, histogram_freq=1)
-        cbks = [tsb]
+    # fit with validation data
+    model.fit(X_train, y_train, batch_size=batch_size,
+              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=2)
 
-        # fit with validation data
-        model.fit(X_train, y_train, batch_size=batch_size,
-                  validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=2)
+    # fit with validation data and accuracy
+    model.fit(X_train, y_train, batch_size=batch_size,
+              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=2)
 
-        # fit with validation data and accuracy
-        model.fit(X_train, y_train, batch_size=batch_size,
-                  validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=2)
+    # fit generator with validation data
+    model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
+                        validation_data=(X_test, y_test),
+                        callbacks=cbks)
 
-        # fit generator with validation data
-        model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
-                            validation_data=(X_test, y_test),
-                            callbacks=cbks)
+    # fit generator without validation data
+    model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
+                        callbacks=cbks)
 
-        # fit generator without validation data
-        model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
-                            callbacks=cbks)
+    # fit generator with validation data and accuracy
+    model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
+                        validation_data=(X_test, y_test),
+                        callbacks=cbks)
 
-        # fit generator with validation data and accuracy
-        model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
-                            validation_data=(X_test, y_test),
-                            callbacks=cbks)
+    # fit generator without validation data and accuracy
+    model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
+                        callbacks=cbks)
 
-        # fit generator without validation data and accuracy
-        model.fit_generator(data_generator(True), len(X_train), nb_epoch=2,
-                            callbacks=cbks)
+    assert os.path.exists(filepath)
+    shutil.rmtree(filepath)
 
-        assert os.path.exists(filepath)
-        shutil.rmtree(filepath)
 
-    # case 2 Graph
+def test_LambdaCallback():
+    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
+                                                         nb_test=test_samples,
+                                                         input_shape=(input_dim,),
+                                                         classification=True,
+                                                         nb_class=nb_class)
+    y_test = np_utils.to_categorical(y_test)
+    y_train = np_utils.to_categorical(y_train)
+    model = Sequential()
+    model.add(Dense(nb_hidden, input_dim=input_dim, activation='relu'))
+    model.add(Dense(nb_class, activation='softmax'))
+    model.compile(loss='categorical_crossentropy',
+                  optimizer='sgd',
+                  metrics=['accuracy'])
 
-    with tf.Graph().as_default():
-        session = tf.Session('')
-        KTF.set_session(session)
-        model = Graph()
-        model.add_input(name='X_vars', input_shape=(input_dim, ))
+    # Start an arbitrary process that should run during model training and be terminated after training has completed.
+    def f():
+        while True:
+            pass
 
-        model.add_node(Dense(nb_hidden, activation="sigmoid"),
-                       name='Dense1', input='X_vars')
-        model.add_node(Dense(nb_class, activation="softmax"),
-                       name='last_dense',
-                       input='Dense1')
-        model.add_output(name='output', input='last_dense')
-        model.compile(optimizer='sgd', loss={'output': 'mse'})
+    p = multiprocessing.Process(target=f)
+    p.start()
+    cleanup_callback = callbacks.LambdaCallback(on_train_end=lambda logs: p.terminate())
 
-        tsb = callbacks.TensorBoard(log_dir=filepath, histogram_freq=1)
-        cbks = [tsb]
+    cbks = [cleanup_callback]
+    model.fit(X_train, y_train, batch_size=batch_size,
+              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=5)
+    p.join()
+    assert not p.is_alive()
 
-        # fit with validation
-        model.fit({'X_vars': X_train, 'output': y_train},
-                  batch_size=batch_size,
-                  validation_data={'X_vars': X_test, 'output': y_test},
-                  callbacks=cbks, nb_epoch=2)
 
-        # fit wo validation
-        model.fit({'X_vars': X_train, 'output': y_train},
-                  batch_size=batch_size,
-                  callbacks=cbks, nb_epoch=2)
+@pytest.mark.skipif((K.backend() != 'tensorflow'),
+                    reason="Requires tensorflow backend")
+def test_TensorBoard_with_ReduceLROnPlateau():
+    import shutil
+    filepath = './logs'
+    (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
+                                                         nb_test=test_samples,
+                                                         input_shape=(input_dim,),
+                                                         classification=True,
+                                                         nb_class=nb_class)
+    y_test = np_utils.to_categorical(y_test)
+    y_train = np_utils.to_categorical(y_train)
 
-        # fit generator with validation
-        model.fit_generator(data_generator_graph(True), 1000, nb_epoch=2,
-                            validation_data={'X_vars': X_test, 'output': y_test},
-                            callbacks=cbks)
+    model = Sequential()
+    model.add(Dense(nb_hidden, input_dim=input_dim, activation='relu'))
+    model.add(Dense(nb_class, activation='softmax'))
+    model.compile(loss='binary_crossentropy',
+                  optimizer='sgd',
+                  metrics=['accuracy'])
 
-        # fit generator wo validation
-        model.fit_generator(data_generator_graph(True), 1000, nb_epoch=2,
-                            callbacks=cbks)
+    cbks = [
+        callbacks.ReduceLROnPlateau(
+            monitor='val_loss',
+            factor=0.5,
+            patience=4,
+            verbose=1),
+        callbacks.TensorBoard(
+            log_dir=filepath)]
 
-        assert os.path.exists(filepath)
-        shutil.rmtree(filepath)
+    model.fit(X_train, y_train, batch_size=batch_size,
+              validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=2)
+
+    assert os.path.exists(filepath)
+    shutil.rmtree(filepath)
 
-    KTF.set_session(old_session)
 
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/test_metrics.py

@@ -34,6 +34,66 @@ def test_metrics():
         assert K.eval(output).shape == ()
 
 
+def test_matthews_correlation():
+    y_true = K.variable(np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0]))
+    y_pred = K.variable(np.array([1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0]))
+
+    # Calculated using sklearn.metrics.matthews_corrcoef
+    expected = -0.14907119849998601
+
+    actual = K.eval(metrics.matthews_correlation(y_true, y_pred))
+    epsilon = 1e-05
+    assert expected - epsilon <= actual <= expected + epsilon
+
+
+def test_precision():
+    y_true = K.variable(np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0]))
+    y_pred = K.variable(np.array([1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0]))
+
+    # Calculated using sklearn.metrics.precision_score
+    expected = 0.40000000000000002
+
+    actual = K.eval(metrics.precision(y_true, y_pred))
+    epsilon = 1e-05
+    assert expected - epsilon <= actual <= expected + epsilon
+
+
+def test_recall():
+    y_true = K.variable(np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0]))
+    y_pred = K.variable(np.array([1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0]))
+
+    # Calculated using sklearn.metrics.recall_score
+    expected = 0.2857142857142857
+
+    actual = K.eval(metrics.recall(y_true, y_pred))
+    epsilon = 1e-05
+    assert expected - epsilon <= actual <= expected + epsilon
+
+
+def test_fbeta_score():
+    y_true = K.variable(np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0]))
+    y_pred = K.variable(np.array([1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0]))
+
+    # Calculated using sklearn.metrics.fbeta_score
+    expected = 0.30303030303030304
+
+    actual = K.eval(metrics.fbeta_score(y_true, y_pred, beta=2))
+    epsilon = 1e-05
+    assert expected - epsilon <= actual <= expected + epsilon
+
+
+def test_fmeasure():
+    y_true = K.variable(np.array([0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0]))
+    y_pred = K.variable(np.array([1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0]))
+
+    # Calculated using sklearn.metrics.f1_score
+    expected = 0.33333333333333331
+
+    actual = K.eval(metrics.fmeasure(y_true, y_pred))
+    epsilon = 1e-05
+    assert expected - epsilon <= actual <= expected + epsilon
+
+
 def test_sparse_metrics():
     for metric in all_sparse_metrics:
         y_a = K.variable(np.random.randint(0, 7, (6,)), dtype=K.floatx())
@@ -41,5 +101,19 @@ def test_sparse_metrics():
         assert K.eval(metric(y_a, y_b)).shape == ()
 
 
+def test_top_k_categorical_accuracy():
+    y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]]))
+    y_true = K.variable(np.array([[0, 1, 0], [1, 0, 0]]))
+    success_result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred,
+                            k=3))
+    assert success_result == 1
+    partial_result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred,
+                            k=2))
+    assert partial_result == 0.5
+    failure_result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred,
+                            k=1))
+    assert failure_result == 0
+
+
 if __name__ == "__main__":
     pytest.main([__file__])

tests/keras/test_sequential_model.py

@@ -6,7 +6,7 @@ import numpy as np
 np.random.seed(1337)
 
 from keras import backend as K
-from keras.models import Graph, Sequential
+from keras.models import Sequential
 from keras.layers.core import Dense, Activation, Merge, Lambda
 from keras.utils import np_utils
 from keras.utils.test_utils import get_test_data, keras_test

tests/test_dynamic_trainability.py

@@ -0,0 +1,114 @@
+from __future__ import absolute_import
+from __future__ import print_function
+import pytest
+
+from keras.utils.test_utils import keras_test
+from keras.models import Model, Sequential
+from keras.layers import Dense, Input
+
+
+@keras_test
+def test_layer_trainability_switch():
+    # with constructor argument, in Sequential
+    model = Sequential()
+    model.add(Dense(2, trainable=False, input_dim=1))
+    assert model.trainable_weights == []
+
+    # by setting the `trainable` argument, in Sequential
+    model = Sequential()
+    layer = Dense(2, input_dim=1)
+    model.add(layer)
+    assert model.trainable_weights == layer.trainable_weights
+    layer.trainable = False
+    assert model.trainable_weights == []
+
+    # with constructor argument, in Model
+    x = Input(shape=(1,))
+    y = Dense(2, trainable=False)(x)
+    model = Model(x, y)
+    assert model.trainable_weights == []
+
+    # by setting the `trainable` argument, in Model
+    x = Input(shape=(1,))
+    layer = Dense(2)
+    y = layer(x)
+    model = Model(x, y)
+    assert model.trainable_weights == layer.trainable_weights
+    layer.trainable = False
+    assert model.trainable_weights == []
+
+
+@keras_test
+def test_model_trainability_switch():
+    # a non-trainable model has no trainable weights
+    x = Input(shape=(1,))
+    y = Dense(2)(x)
+    model = Model(x, y)
+    model.trainable = False
+    assert model.trainable_weights == []
+
+    # same for Sequential
+    model = Sequential()
+    model.add(Dense(2, input_dim=1))
+    model.trainable = False
+    assert model.trainable_weights == []
+
+
+@keras_test
+def test_nested_model_trainability():
+    # a Sequential inside a Model
+    inner_model = Sequential()
+    inner_model.add(Dense(2, input_dim=1))
+
+    x = Input(shape=(1,))
+    y = inner_model(x)
+    outer_model = Model(x, y)
+    assert outer_model.trainable_weights == inner_model.trainable_weights
+    inner_model.trainable = False
+    assert outer_model.trainable_weights == []
+    inner_model.trainable = True
+    inner_model.layers[-1].trainable = False
+    assert outer_model.trainable_weights == []
+
+    # a Sequential inside a Sequential
+    inner_model = Sequential()
+    inner_model.add(Dense(2, input_dim=1))
+    outer_model = Sequential()
+    outer_model.add(inner_model)
+    assert outer_model.trainable_weights == inner_model.trainable_weights
+    inner_model.trainable = False
+    assert outer_model.trainable_weights == []
+    inner_model.trainable = True
+    inner_model.layers[-1].trainable = False
+    assert outer_model.trainable_weights == []
+
+    # a Model inside a Model
+    x = Input(shape=(1,))
+    y = Dense(2)(x)
+    inner_model = Model(x, y)
+    x = Input(shape=(1,))
+    y = inner_model(x)
+    outer_model = Model(x, y)
+    assert outer_model.trainable_weights == inner_model.trainable_weights
+    inner_model.trainable = False
+    assert outer_model.trainable_weights == []
+    inner_model.trainable = True
+    inner_model.layers[-1].trainable = False
+    assert outer_model.trainable_weights == []
+
+    # a Model inside a Sequential
+    x = Input(shape=(1,))
+    y = Dense(2)(x)
+    inner_model = Model(x, y)
+    outer_model = Sequential()
+    outer_model.add(inner_model)
+    assert outer_model.trainable_weights == inner_model.trainable_weights
+    inner_model.trainable = False
+    assert outer_model.trainable_weights == []
+    inner_model.trainable = True
+    inner_model.layers[-1].trainable = False
+    assert outer_model.trainable_weights == []
+
+
+if __name__ == '__main__':
+    pytest.main([__file__])

tests/test_loss_weighting.py

@@ -5,7 +5,7 @@ import numpy as np
 np.random.seed(1337)
 
 from keras.utils.test_utils import get_test_data
-from keras.models import Sequential, Graph
+from keras.models import Sequential
 from keras.layers import Dense, Activation, RepeatVector, TimeDistributedDense, GRU
 from keras.utils import np_utils
 from keras.utils.test_utils import keras_test