Prepare 1.0.5 PyPI release

* added masking to merge layer (#2413)

* added documentation, fixed stylistic issues

* removed casting

* changed to using K.all

.travis.yml

@@ -57,6 +57,8 @@ install:
 script:
   # run keras backend init to initialize backend config
   - python -c "import keras.backend"
+  # create dataset directory to avoid concurrent directory creation at runtime
+  - mkdir ~/.keras/datasets
   # set up keras backend
   - sed -i -e 's/"backend":[[:space:]]*"[^"]*/"backend":\ "'$KERAS_BACKEND'/g' ~/.keras/keras.json;
   - echo -e "Running tests with the following config:\n$(cat ~/.keras/keras.json)"

README.md

@@ -38,7 +38,7 @@ Keras is compatible with: __Python 2.7-3.5__.
 
 ## Getting started: 30 seconds to Keras
 
-The core data structure of Keras is a __model__, a way to organize layers. The main type of model is the [`Sequential`](http://keras.io/getting-started/sequential-model-guide) model, a linear stack of layers. For more complex architectures, you should use the [Keras function API](http://keras.io/getting-started/functional-api-guide).
+The core data structure of Keras is a __model__, a way to organize layers. The main type of model is the [`Sequential`](http://keras.io/getting-started/sequential-model-guide) model, a linear stack of layers. For more complex architectures, you should use the [Keras functional API](http://keras.io/getting-started/functional-api-guide).
 
 Here's the `Sequential` model:
 
@@ -51,7 +51,7 @@ model = Sequential()
 Stacking layers is as easy as `.add()`:
 
 ```python
-from keras.layers.core import Dense, Activation
+from keras.layers import Dense, Activation
 
 model.add(Dense(output_dim=64, input_dim=100))
 model.add(Activation("relu"))

docs/autogen.py

@@ -53,10 +53,16 @@ Scikit-learn API
 
 '''
 from __future__ import print_function
+from __future__ import unicode_literals
+
 import re
 import inspect
 import os
 import shutil
+import sys
+if sys.version[0] == '2':
+    reload(sys)
+    sys.setdefaultencoding('utf8')
 
 from keras.layers import convolutional
 from keras.layers import recurrent
@@ -76,6 +82,7 @@ from keras import constraints
 from keras import activations
 from keras import regularizers
 
+
 EXCLUDE = {
     'Optimizer',
     'Wrapper',
@@ -250,8 +257,6 @@ def get_function_signature(function, method=True):
     for a, v in kwargs:
         if type(v) == str:
             v = '\'' + v + '\''
-        elif type(v) == unicode:
-            v = 'u\'' + v + '\''
         st += str(a) + '=' + str(v) + ', '
     if kwargs or args:
         return st[:-2] + ')'
@@ -330,6 +335,7 @@ def process_function_docstring(docstring):
 print('Cleaning up existing sources directory.')
 if os.path.exists('sources'):
     shutil.rmtree('sources')
+
 print('Populating sources directory with templates.')
 for subdir, dirs, fnames in os.walk('templates'):
     for fname in fnames:
@@ -414,103 +420,3 @@ for page_data in PAGES:
     if not os.path.exists(subdir):
         os.makedirs(subdir)
     open(path, 'w').write(mkdown)
-
-
-# covered_so_far = set()
-# for module, module_name in MODULES:
-#     class_pages = []
-#     for name in dir(module):
-#         if name in SKIP:
-#             continue
-#         if name[0] == '_':
-#             continue
-#         module_member = getattr(module, name)
-#         if module_member in covered_so_far:
-#             continue
-#         if inspect.isclass(module_member):
-#             cls = module_member
-#             if cls.__module__ == module_name:
-
-#                 try:
-#                     class_signature = get_function_signature(cls.__init__)
-#                     class_signature = class_signature.replace('__init__', cls.__name__)
-#                 except:
-#                     # in case the class inherits from object and does not
-#                     # define __init__
-#                     class_signature = module_name + '.' + cls.__name__ + '()'
-
-#                 functions = []
-#                 functions_not_defined_here = []
-#                 for name in dir(cls):
-#                     if name in SKIP:
-#                         continue
-#                     if name[0] == '_':
-#                         continue
-#                     cls_member = getattr(cls, name)
-#                     if inspect.isfunction(cls_member):
-#                         function = cls_member
-#                         signature = inspect.getargspec(function)
-#                         defaults = signature.defaults
-#                         args = signature.args[1:]
-#                         if defaults:
-#                             kwargs = zip(args[-len(defaults):], defaults)
-#                             args = args[:-len(defaults)]
-#                         else:
-#                             kwargs = []
-
-#                         defined_by = get_earliest_class_that_defined_member(function.__name__, cls)
-#                         if cls == defined_by:
-#                             functions.append(function)
-#                         else:
-#                             functions_not_defined_here.append((function, defined_by))
-
-#                 blocks = []
-#                 blocks.append('<span style="float:right;">' + class_to_source_link(cls) + '</span>')
-#                 blocks.append('# ' + cls.__name__ + '\n')
-#                 blocks.append(code_snippet(class_signature))
-#                 docstring = cls.__doc__
-#                 if docstring:
-#                     blocks.append(process_class_docstring(docstring))
-
-#                 if cls.__name__ in INCLUDE_functionS_FOR:
-#                     if functions or functions_not_defined_here:
-#                         blocks.append('### functions\n')
-#                         for function in functions:
-#                             signature = get_function_signature(function)
-#                             signature = signature.replace(module_name + '.', '')
-#                             blocks.append(code_snippet(signature))
-#                             docstring = function.__doc__
-#                             if docstring:
-#                                 blocks.append(process_function_docstring(docstring))
-#                         for function, defined_by in functions_not_defined_here:
-#                             signature = get_function_signature(function)
-#                             function_module_name = function.__module__
-#                             signature = signature.replace(function_module_name + '.', '')
-#                             link = '[' + defined_by.__name__ + '](' + class_to_docs_link(defined_by) + ')'
-#                             blocks.append(code_snippet(signature))
-#                             blocks.append('Defined by ' + link + '.\n')
-
-#                 mkdown = '\n'.join(blocks)
-#                 class_pages.append((id(cls), mkdown))
-#                 covered_so_far.add(module_member)
-
-#     class_pages.sort(key=lambda x: x[0])
-#     class_pages = [x[1] for x in class_pages]
-#     module_page = '\n----\n\n'.join(class_pages)
-
-#     # save module page.
-#     # Either insert content into existing page,
-#     # or create page otherwise
-#     path = 'sources/' + module_name.replace('.', '/')[6:] + '.md'
-#     if os.path.exists(path):
-#         template = open(path).read()
-#         assert '{{autogenerated}}' in template, ('Template found for ' + path +
-#                                                  ' but missing {{autogenerated}} tag.')
-#         module_page = template.replace('{{autogenerated}}', module_page)
-#         print('...inserting autogenerated content into template:', path)
-#     else:
-#         print('...creating new page with autogenerated content:', path)
-#     subdir = os.path.dirname(path)
-#     if not os.path.exists(subdir):
-#         os.makedirs(subdir)
-#     open(path, 'w').write(module_page)

docs/templates/activations.md

@@ -30,6 +30,7 @@ model.add(Activation(tanh))
 
 - __softmax__: Softmax applied across inputs last dimension. Expects shape either `(nb_samples, nb_timesteps, nb_dims)` or `(nb_samples, nb_dims)`.
 - __softplus__
+- __softsign__
 - __relu__
 - __tanh__
 - __sigmoid__

docs/templates/getting-started/faq.md

@@ -10,6 +10,7 @@
 - [How is the validation split computed?](#how-is-the-validation-split-computed)
 - [Is the data shuffled during training?](#is-the-data-shuffled-during-training)
 - [How can I record the training / validation loss / accuracy at each epoch?](#how-can-i-record-the-training-validation-loss-accuracy-at-each-epoch)
+- [How can I "freeze" layers?](#how-can-i-freeze-keras-layers)
 - [How can I use stateful RNNs?](#how-can-i-use-stateful-rnns)
 
 ---
@@ -20,12 +21,11 @@ Please cite Keras in your publications if it helps your research. Here is an exa
 
 ```
 @misc{chollet2015keras,
-  author = {Chollet, François},
-  title = {Keras},
-  year = {2015},
-  publisher = {GitHub},
-  journal = {GitHub repository},
-  howpublished = {\url{https://github.com/fchollet/keras}}
+  title={Keras},
+  author={Chollet, Fran\c{c}ois},
+  year={2015},
+  publisher={GitHub},
+  howpublished={\url{https://github.com/fchollet/keras}},
 }
 ```
 
@@ -102,6 +102,11 @@ model = model_from_json(open('my_model_architecture.json').read())
 model.load_weights('my_model_weights.h5')
 ```
 
+Finally, before it can be used, the model shall be compiled.
+```python
+model.compile(optimizer='adagrad', loss='mse')
+```
+
 ---
 
 ### Why is the training loss much higher than the testing loss?
@@ -134,14 +139,28 @@ to pass the learning phase flag to your function:
 get_3rd_layer_output = K.function([model.layers[0].input, K.learning_phase()],
                                   [model.layers[3].output])
 
-# output in train mode = 0
+# output in test mode = 0
 layer_output = get_3rd_layer_output([X, 0])[0]
 
-# output in test mode = 1
+# output in train mode = 1
 layer_output = get_3rd_layer_output([X, 1])[0]
 ```
 
-Another more flexible way of getting output from intermediate layers is to use the [functional API](/getting-started/functional-api-guide).
+Another more flexible way of getting output from intermediate layers is to use the [functional API](/getting-started/functional-api-guide). For example, if you have created an autoencoder for MNIST:
+
+```python
+inputs = Input(shape=(784,))
+encoded = Dense(32, activation='relu')(inputs)
+decoded = Dense(784)(encoded)
+model = Model(input=inputs, output=decoded)
+```
+
+After compiling and training the model, you can get the output of the data from the encoder like this:
+
+```python
+encoder = Model(input=inputs, output=encoded)
+X_encoded = encoder.predict(X)
+```
 
 ---
 
@@ -196,6 +215,40 @@ print(hist.history)
 
 ---
 
+### How can I "freeze" Keras layers?
+
+To "freeze" a layer means to exclude it from training, i.e. its weights will never be updated. This is useful in the context of fine-tuning a model, or using fixed embeddings for a text input.
+
+You can pass a `trainable` argument (boolean) to a layer constructor to set a layer to be non-trainable:
+
+```python
+frozen_layer = Dense(32, trainable=False)
+```
+
+Additionally, you can set the `trainable` property of a layer to `True` or `False` after instantiation. For this to take effect, you will need to call `compile()` on your model after modifying the `trainable` property. Here's an example:
+
+```python
+x = Input(shape=(32,))
+layer = Dense(32)
+layer.trainable = False
+y = layer(x)
+
+frozen_model = Model(x, y)
+# in the model below, the weights of `layer` will not be updated during training
+frozen_model.compile(optimizer='rmsprop', loss='mse')
+
+layer.trainable = True
+trainable_model = Model(x, y)
+# with this model the weights of the layer will be updated during training
+# (which will also affect the above model since it uses the same layer instance)
+trainable_model.compile(optimizer='rmsprop', loss='mse')
+
+frozen_model.fit(data, labels)  # this does NOT update the weights of `layer`
+trainable_model.fit(data, labels)  # this updates the weights of `layer`
+```
+
+---
+
 ### How can I use stateful RNNs?
 
 Making a RNN stateful means that the states for the samples of each batch will be reused as initial states for the samples in the next batch.

docs/templates/getting-started/functional-api-guide.md

@@ -166,7 +166,7 @@ Let's consider a dataset of tweets. We want to build a model that can tell wheth
 
 One way to achieve this is to build a model that encodes two tweets into two vectors, concatenates the vectors and adds a logistic regression of top, outputting a probability that the two tweets share the same author. The model would then be trained on positive tweet pairs and negative tweet pairs.
 
-Because the problem is symetric, the mechanism that encodes the first tweet should be reused (weights and all) to encode the second tweet. Here we use a shared LSTM layer to encode the tweets.
+Because the problem is symmetric, the mechanism that encodes the first tweet should be reused (weights and all) to encode the second tweet. Here we use a shared LSTM layer to encode the tweets.
 
 Let's build this with the functional API. We will take as input for a tweet a binary matrix of shape `(140, 256)`, i.e. a sequence of 140 vectors of size 256, where each dimension in the 256-dimensional vector encodes the presence/absence of a character (out of an alphabet of 256 frequent characters).
 
@@ -309,8 +309,8 @@ from keras.layers import merge, Convolution2D, Input
 
 # input tensor for a 3-channel 256x256 image
 x = Input(shape=(3, 256, 256))
-# 3x3 conv with 16 output channels
-y = Convolution2D(16, 3, 3, border_mode='same')
+# 3x3 conv with 3 output channels (same as input channels)
+y = Convolution2D(3, 3, 3, border_mode='same')
 # this returns x + y.
 z = merge([x, y], mode='sum')
 ```

docs/templates/getting-started/sequential-model-guide.md

@@ -6,6 +6,7 @@ You can create a `Sequential` model by passing a list of layer instances to the
 
 ```python
 from keras.models import Sequential
+from keras.layers import Dense, Activation
 
 model = Sequential([
     Dense(32, input_dim=784),
@@ -87,6 +88,13 @@ final_model.add(Dense(10, activation='softmax'))
 
 <img src="http://s3.amazonaws.com/keras.io/img/two_branches_sequential_model.png" alt="two branch Sequential" style="width: 400px;"/>
 
+Such a two-branch model can then be trained via e.g.:
+
+```python
+final_model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
+final_model.fit([input_data_1, input_data_2], targets)  # we pass one data array per model input
+```
+
 The `Merge` layer supports a number of pre-defined modes:
 
 - `sum` (default): element-wise sum
@@ -112,7 +120,7 @@ Now you know enough to be able to define *almost* any model with Keras. For comp
 Before training a model, you need to configure the learning process, which is done via the `compile` method. It receives three arguments:
 
 - an optimizer. This could be the string identifier of an existing optimizer (such as `rmsprop` or `adagrad`), or an instance of the `Optimizer` class. See: [optimizers](/optimizers).
-- a loss function. This is the objective that the model will try to minimize. If can be the string identifier of an existing loss function (such as `categorical_crossentropy` or `mse`), or it can be an objective function. See: [objectives](/objectives).
+- a loss function. This is the objective that the model will try to minimize. It can be the string identifier of an existing loss function (such as `categorical_crossentropy` or `mse`), or it can be an objective function. See: [objectives](/objectives).
 - a list of metrics. For any classification problem you will want to set this to `metrics=['accuracy']`. A metric could be the string identifier of an existing metric (only `accuracy` is supported at this point), or a custom metric function.
 
 ```python
@@ -538,4 +546,4 @@ y_val = np.random.random((100, nb_classes))
 decoder.fit([x_train_a, x_train_b], y_train,
             batch_size=64, nb_epoch=5,
             validation_data=([x_val_a, x_val_b], y_val))
-```
+```

docs/templates/index.md

@@ -36,7 +36,7 @@ Keras is compatible with: __Python 2.7-3.5__.
 
 ## Getting started: 30 seconds to Keras
 
-The core data structure of Keras is a __model__, a way to organize layers. The main type of model is the [`Sequential`](http://keras.io/getting-started/sequential-model-guide) model, a linear stack of layers. For more complex architectures, you should use the [Keras function API](http://keras.io/getting-started/functional-api-guide).
+The core data structure of Keras is a __model__, a way to organize layers. The main type of model is the [`Sequential`](http://keras.io/getting-started/sequential-model-guide) model, a linear stack of layers. For more complex architectures, you should use the [Keras functional API](http://keras.io/getting-started/functional-api-guide).
 
 Here's the `Sequential` model:
 
@@ -49,7 +49,7 @@ model = Sequential()
 Stacking layers is as easy as `.add()`:
 
 ```python
-from keras.layers.core import Dense, Activation
+from keras.layers import Dense, Activation
 
 model.add(Dense(output_dim=64, input_dim=100))
 model.add(Activation("relu"))

docs/templates/layers/writing-your-own-keras-layers.md

@@ -0,0 +1,34 @@
+# Writing your own Keras layers
+
+For simple, stateless custom operations, you are probably better off using `layers.core.Lambda` layers. But for any custom operation that has trainable weights, you should implement your own layer.
+
+Here is the skeleton of a Keras layer. There are only three methods you need to implement:
+
+- `build(input_shape)`: this is where you will define your weights. Trainable weights should be added to the list `self.trainable_weights`. Other attributes of note are: `self.non_trainable_weights` (list) and `self.updates` (list of update tuples (tensor, new_tensor)). For an example of how to use `non_trainable_weights` and `updates`, see the code for the `BatchNormalization` layer.
+- `call(x)`: this is where the layer's logic lives. Unless you want your layer to support masking, you only have to care about the first argument passed to `call`: the input tensor.
+- `get_output_shape_for(input_shape)`: in case your layer modifies the shape of its input, you should specify here the shape transformation logic. This allows Keras to do automatic shape inference.
+
+```python
+from keras import backend as K
+from keras.engine.topology import Layer
+import numpy as np
+
+class MyLayer(Layer):
+    def __init__(self, output_dim, **kwargs):
+        self.output_dim = output_dim
+        super(MyLayer, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        input_dim = input_shape[1]
+        initial_weight_value = np.random.random((input_dim, output_dim))
+        self.W = K.variable(initial_weight_value)
+        self.trainable_weights = [self.W]
+
+    def call(self, x, mask=None):
+        return K.dot(x, self.W)
+
+    def get_output_shape_for(self, input_shape):
+        return (input_shape[0], self.output_dim)
+```
+
+The existing Keras layers provide ample examples of how to implement almost anything. Never hesitate to read the source code!

docs/templates/objectives.md

@@ -26,5 +26,7 @@ For a few examples of such functions, check out the [objectives source](https://
 - __hinge__
 - __binary_crossentropy__: Also known as logloss. 
 - __categorical_crossentropy__: Also known as multiclass logloss. __Note__: using this objective requires that your labels are binary arrays of shape `(nb_samples, nb_classes)`.
-- __poisson__: mean of `(predictions - targets * log(predictions))`
-- __cosine_proximity__: the opposite (negative) of the mean cosine proximity between predictions and targets.
+- __sparse_categorical_crossentropy__: As above but accepts sparse labels. __Note__: this objective still requires that your labels have the same number of dimensions as your outputs; you may need to add a length-1 dimension to the shape of your labels, e.g with `np.expand_dims(y, -1)`.
+- __kullback_leibler_divergence__ / __kld__: Information gain from a predicted probability distribution Q to a true probability distribution P. Gives a measure of difference between both distributions.
+- __poisson__: Mean of `(predictions - targets * log(predictions))`
+- __cosine_proximity__: The opposite (negative) of the mean cosine proximity between predictions and targets.

docs/templates/preprocessing/image.md

@@ -2,18 +2,23 @@
 ## ImageDataGenerator
 
 ```python
-keras.preprocessing.image.ImageDataGenerator(featurewise_center=True,
+keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
     samplewise_center=False,
-    featurewise_std_normalization=True,
+    featurewise_std_normalization=False,
     samplewise_std_normalization=False,
     zca_whitening=False,
     rotation_range=0.,
     width_shift_range=0.,
     height_shift_range=0.,
     shear_range=0.,
+    zoom_range=0.,
+    channel_shift_range=0.,
+    fill_mode='nearest',
+    cval=0.,
     horizontal_flip=False,
     vertical_flip=False,
-    dim_ordering='th')
+    rescale=None,
+    dim_ordering=K.image_dim_ordering())
 ```
 
 Generate batches of tensor image data with real-time data augmentation. The data will be looped over (in batches) indefinitely.
@@ -28,29 +33,60 @@ Generate batches of tensor image data with real-time data augmentation. The data
     - __width_shift_range__: Float (fraction of total width). Range for random horizontal shifts.
     - __height_shift_range__: Float (fraction of total height). Range for random vertical shifts.
     - __shear_range__: Float. Shear Intensity (Shear angle in counter-clockwise direction as radians)
+    - __zoom_range__: Float or [lower, upper]. Range for random zoom. If a float, `[lower, upper] = [1-zoom_range, 1+zoom_range]`.
+    - __channel_shift_range__: Float. Range for random channel shifts.
+    - __fill_mode__: One of {"constant", "nearest", "reflect" or "wrap"}.  Points outside the boundaries of the input are filled according to the given mode.
+    - __cval__: Float or Int. Value used for points outside the boundaries when `fill_mode = "constant"`.
     - __horizontal_flip__: Boolean. Randomly flip inputs horizontally.
     - __vertical_flip__: Boolean. Randomly flip inputs vertically.
+    - __rescale__: rescaling factor. Defaults to None. If None or 0, no rescaling is applied,
+            otherwise we multiply the data by the value provided (before applying
+            any other transformation).
     - __dim_ordering__: One of {"th", "tf"}.
         "tf" mode means that the images should have shape `(samples, width, height, channels)`,
         "th" mode means that the images should have shape `(samples, channels, width, height)`.
+        It defaults to the `image_dim_ordering` value found in your
+        Keras config file at `~/.keras/keras.json`.
+        If you never set it, then it will be "th".
 
 - __Methods__:
-    - __fit(X)__: Required if featurewise_center or featurewise_std_normalization or zca_whitening. Compute necessary quantities on some sample data.
+    - __fit(X)__: Compute the internal data stats related to the data-dependent transformations, based on an array of sample data.
+        Only required if featurewise_center or featurewise_std_normalization or zca_whitening.
         - __Arguments__:
             - __X__: sample data.
             - __augment__: Boolean (default: False). Whether to fit on randomly augmented samples.
             - __rounds__: int (default: 1). If augment, how many augmentation passes over the data to use.
-    - __flow(X, y)__:
+    - __flow(X, y)__: Takes numpy data & label arrays, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
         - __Arguments__:
             - __X__: data.
             - __y__: labels.
             - __batch_size__: int (default: 32).
             - __shuffle__: boolean (defaut: False).
-            - __save_to_dir__: None or str. This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
-            - __save_prefix__: str. Prefix to use for filenames of saved pictures.
-            - __save_format__: one of "png", jpeg".
+            - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
+            - __save_prefix__: str (default: `''`). Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
+            - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "jpeg".
+        - ___yields__: Tuples of `(x, y)` where `x` is a numpy array of image data and `y` is a numpy array of corresponding labels.
+            The generator loops indefinitely.
+    - __flow_from_directory(directory)__: Takes the path to a directory, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
+        - __Arguments__:
+            - __directory: path to the target directory. It should contain one subdirectory per class,
+                and the subdirectories should contain PNG or JPG images. See [this script](https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d) for more details.
+            - __target_size__: tuple of integers, default: `(256, 256)`. The dimensions to which all images found will be resized.
+            - __color_mode__: one of "grayscale", "rbg". Default: "rgb". Whether the images will be converted to have 1 or 3 color channels.
+            - __classes__: optional list of class subdirectories (e.g. `['dogs', 'cats']`). Default: None. If not provided, the list of classes will be automatically inferred (and the order of the classes, which will map to the label indices, will be alphanumeric).
+            - __class_mode__: one of "categorical", "binary", "sparse" or None. Default: "categorical". Determines the type of label arrays that are returned: "categorical" will be 2D one-hot encoded labels, "binary" will be 1D binary labels, "sparse" will be 1D integer labels. If None, no labels are returned (the generator will only yield batches of image data, which is useful to use `model.predict_generator()`, `model.evaluate_generator()`, etc.).
+            - __batch_size__: size of the batches of data (default: 32).
+            - __shuffle__: whether to shuffle the data (default: True)
+            - __seed__: optional random seed for shuffling.
+            - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
+            - __save_prefix__: str. Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
+            - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "jpeg".
+
+
+- __Examples__:
+
+Example of using `.flow(X, y)`:
 
-- __Example__:
 ```python
 (X_train, y_train), (X_test, y_test) = cifar10.load_data(test_split=0.1)
 Y_train = np_utils.to_categorical(y_train, nb_classes)
@@ -84,3 +120,34 @@ for e in range(nb_epoch):
             # the generator loops indefinitely
             break
 ```
+
+Example of using `.flow_from_directory(directory)`:
+
+```python
+train_datagen = ImageDataGenerator(
+        rescale=1./255,
+        shear_range=0.2,
+        zoom_range=0.2,
+        horizontal_flip=True)
+
+test_datagen = ImageDataGenerator(rescale=1./255)
+
+train_generator = train_datagen.flow_from_directory(
+        'data/train',
+        target_size=(150, 150),
+        batch_size=32,
+        class_mode='binary')
+
+validation_generator = test_datagen.flow_from_directory(
+        'data/validation',
+        target_size=(150, 150),
+        batch_size=32,
+        class_mode='binary')
+
+model.fit_generator(
+        train_generator,
+        samples_per_epoch=2000,
+        nb_epoch=50,
+        validation_data=validation_generator,
+        nb_val_samples=800)
+```

docs/templates/preprocessing/sequence.md

@@ -4,14 +4,14 @@
 keras.preprocessing.sequence.pad_sequences(sequences, maxlen=None, dtype='int32')
 ```
 
-Transform a list of `nb_samples sequences` (lists of scalars) into a 2D numpy array of shape `(nb_samples, nb_timesteps)`. `nb_timesteps` is either the `maxlen` argument if provided, or the length of the longest sequence otherwise. Sequences that are shorter than `nb_timesteps` are padded with zeros at the end.
+Transform a list of `nb_samples sequences` (lists of scalars) into a 2D Numpy array of shape `(nb_samples, nb_timesteps)`. `nb_timesteps` is either the `maxlen` argument if provided, or the length of the longest sequence otherwise. Sequences that are shorter than `nb_timesteps` are padded with zeros at the end.
 
-- __Return__: 2D numpy array of shape `(nb_samples, nb_timesteps)`.
+- __Return__: 2D Numpy array of shape `(nb_samples, nb_timesteps)`.
 
 - __Arguments__:
     - __sequences__: List of lists of int or float.
     - __maxlen__: None or int. Maximum sequence length, longer sequences are truncated and shorter sequences are padded with zeros at the end.
-    - __dtype__: datatype of the numpy array returned.
+    - __dtype__: datatype of the Numpy array returned.
     - __padding__: 'pre' or 'post', pad either before or after each sequence.
     - __truncating__: 'pre' or 'post', remove values from sequences larger than maxlen either in the beginning or in the end of the sequence
     - __value__: float, value to pad the sequences to the desired value.
@@ -21,12 +21,12 @@ Transform a list of `nb_samples sequences` (lists of scalars) into a 2D numpy ar
 ## skipgrams
 
 ```python
-keras.preprocessing.sequence.skipgrams(sequence, vocabulary_size, 
-    window_size=4, negative_samples=1., shuffle=True, 
+keras.preprocessing.sequence.skipgrams(sequence, vocabulary_size,
+    window_size=4, negative_samples=1., shuffle=True,
     categorical=False, sampling_table=None)
 ```
 
-Transforms a sequence of word indexes (list of int) into couples of the form: 
+Transforms a sequence of word indexes (list of int) into couples of the form:
 
 - (word, word in the same window), with label 1 (positive samples).
 - (word, random word from the vocabulary), with label 0 (negative samples).
@@ -34,8 +34,8 @@ Transforms a sequence of word indexes (list of int) into couples of the form:
 Read more about Skipgram in this gnomic paper by Mikolov et al.: [Efficient Estimation of Word Representations in
 Vector Space](http://arxiv.org/pdf/1301.3781v3.pdf)
 
-- __Return__: tuple `(couples, labels)`. 
-    - `couples` is a list of 2-elements lists of int: `[word_index, other_word_index]`. 
+- __Return__: tuple `(couples, labels)`.
+    - `couples` is a list of 2-elements lists of int: `[word_index, other_word_index]`.
     - `labels` is a list of 0 and 1, where 1 indicates that `other_word_index` was found in the same window as `word_index`, and 0 indicates that `other_word_index` was random.
     - if categorical is set to True, the labels are categorical, ie. 1 becomes [0,1], and 0 becomes [1, 0].
 
@@ -46,7 +46,7 @@ Vector Space](http://arxiv.org/pdf/1301.3781v3.pdf)
     - __negative_samples__: float >= 0. 0 for no negative (=random) samples. 1 for same number as positive samples. etc.
     - __shuffle__: boolean. Whether to shuffle the samples.
     - __categorical__: boolean. Whether to make the returned labels categorical.
-    - __sampling_table__: numpy array of shape `(vocabulary_size,)` where `sampling_table[i]` is the probability of sampling the word with index i (assumed to be i-th most common word in the dataset).
+    - __sampling_table__: Numpy array of shape `(vocabulary_size,)` where `sampling_table[i]` is the probability of sampling the word with index i (assumed to be i-th most common word in the dataset).
 
 
 ---
@@ -59,7 +59,7 @@ keras.preprocessing.sequence.make_sampling_table(size, sampling_factor=1e-5)
 
 Used for generating the `sampling_table` argument for `skipgrams`. `sampling_table[i]` is the probability of sampling the word i-th most common word in a dataset (more common words should be sampled less frequently, for balance).
 
-- __Return__: numpy array of shape `(size,)`.
+- __Return__: Numpy array of shape `(size,)`.
 
 - __Arguments__:
     - __size__: size of the vocabulary considered.

docs/templates/scikit-learn-api.md

@@ -1,4 +1,4 @@
-# Wrappers for the Sciki-Learn API
+# Wrappers for the Scikit-Learn API
 
 You can use `Sequential` Keras models (single-input only) as part of your Scikit-Learn workflow via the wrappers found at `keras.wrappers.sklearn.py`.
 
@@ -25,7 +25,7 @@ present class will then be treated as the default build_fn.
 
 `sk_params` takes both model parameters and fitting parameters. Legal model
 parameters are the arguments of `build_fn`. Note that like all other
-estimators in scikit-learn, 'build_fn' should provide defalult values for
+estimators in scikit-learn, 'build_fn' should provide default values for
 its arguments, so that you could create the estimator without passing any
 values to `sk_params`.
 
@@ -42,4 +42,4 @@ fitting (predicting) parameters are selected in the following order:
 When using scikit-learn's `grid_search` API, legal tunable parameters are
 those you could pass to `sk_params`, including fitting parameters.
 In other words, you could use `grid_search` to search for the best
-`batch_size` or `nb_epoch` as well as the model parameters.
+`batch_size` or `nb_epoch` as well as the model parameters.

docs/templates/visualization.md

@@ -10,9 +10,10 @@ from keras.utils.visualize_util import plot
 plot(model, to_file='model.png')
 ```
 
-`plot` takes one optional arguments:
+`plot` takes two optional arguments:
 
 - `show_shapes` (defaults to False) controls whether output shapes are shown in the graph.
+- `show_layer_names` (defaults to True) controls whether layer names are shown in the graph.
 
 You can also directly obtain the `pydot.Graph` object and render it yourself,
 for example to show it in an ipython notebook :

examples/addition_rnn.py

@@ -29,8 +29,7 @@ Five digits inverted:
 from __future__ import print_function
 from keras.models import Sequential
 from keras.engine.training import slice_X
-from keras.layers.core import Activation, TimeDistributedDense, RepeatVector
-from keras.layers import recurrent
+from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, recurrent
 import numpy as np
 from six.moves import range
 
@@ -40,7 +39,7 @@ class CharacterTable(object):
     Given a set of characters:
     + Encode them to a one hot integer representation
     + Decode the one hot integer representation to their character output
-    + Decode a vector of probabilties to their character output
+    + Decode a vector of probabilities to their character output
     '''
     def __init__(self, chars, maxlen):
         self.chars = sorted(set(chars))
@@ -140,7 +139,7 @@ for _ in range(LAYERS):
     model.add(RNN(HIDDEN_SIZE, return_sequences=True))
 
 # For each of step of the output sequence, decide which character should be chosen
-model.add(TimeDistributedDense(len(chars)))
+model.add(TimeDistributed(Dense(len(chars))))
 model.add(Activation('softmax'))
 
 model.compile(loss='categorical_crossentropy',

examples/antirectifier.py

@@ -2,7 +2,7 @@
 
 We build a custom activation layer called 'Antirectifier',
 which modifies the shape of the tensor that passes through it.
-We need to specify two methods: `output_shape` and `get_output`.
+We need to specify two methods: `get_output_shape_for` and `call`.
 
 Note that the same result can also be achieved via a Lambda layer.
 
@@ -12,7 +12,7 @@ backend (`K`), our code can run both on TensorFlow and Theano.
 
 from __future__ import print_function
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Layer, Activation
+from keras.layers import Dense, Dropout, Layer, Activation
 from keras.datasets import mnist
 from keras import backend as K
 from keras.utils import np_utils

examples/babi_memnn.py

@@ -16,8 +16,8 @@ Time per epoch: 3s on CPU (core i7).
 from __future__ import print_function
 from keras.models import Sequential
 from keras.layers.embeddings import Embedding
-from keras.layers.core import Activation, Dense, Merge, Permute, Dropout
-from keras.layers.recurrent import LSTM
+from keras.layers import Activation, Dense, Merge, Permute, Dropout
+from keras.layers import LSTM
 from keras.utils.data_utils import get_file
 from keras.preprocessing.sequence import pad_sequences
 from functools import reduce
@@ -94,8 +94,13 @@ def vectorize_stories(data, word_idx, story_maxlen, query_maxlen):
             pad_sequences(Xq, maxlen=query_maxlen), np.array(Y))
 
 
-path = get_file('babi-tasks-v1-2.tar.gz',
-                origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz')
+try:
+    path = get_file('babi-tasks-v1-2.tar.gz', origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz')
+except:
+    print('Error downloading dataset, please download it manually:\n'
+          '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'
+          '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')
+    raise
 tar = tarfile.open(path)
 
 challenges = {

examples/babi_rnn.py

@@ -66,7 +66,7 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.utils.data_utils import get_file
 from keras.layers.embeddings import Embedding
-from keras.layers.core import Dense, Merge, Dropout, RepeatVector
+from keras.layers import Dense, Merge, Dropout, RepeatVector
 from keras.layers import recurrent
 from keras.models import Sequential
 from keras.preprocessing.sequence import pad_sequences
@@ -146,7 +146,13 @@ BATCH_SIZE = 32
 EPOCHS = 40
 print('RNN / Embed / Sent / Query = {}, {}, {}, {}'.format(RNN, EMBED_HIDDEN_SIZE, SENT_HIDDEN_SIZE, QUERY_HIDDEN_SIZE))
 
-path = get_file('babi-tasks-v1-2.tar.gz', origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz')
+try:
+    path = get_file('babi-tasks-v1-2.tar.gz', origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz')
+except:
+    print('Error downloading dataset, please download it manually:\n'
+          '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'
+          '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')
+    raise
 tar = tarfile.open(path)
 # Default QA1 with 1000 samples
 # challenge = 'tasks_1-20_v1-2/en/qa1_single-supporting-fact_{}.txt'

examples/cifar10_cnn.py

@@ -15,8 +15,8 @@ from __future__ import print_function
 from keras.datasets import cifar10
 from keras.preprocessing.image import ImageDataGenerator
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Flatten
-from keras.layers.convolutional import Convolution2D, MaxPooling2D
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Convolution2D, MaxPooling2D
 from keras.optimizers import SGD
 from keras.utils import np_utils
 

examples/deep_dream.py

@@ -9,7 +9,7 @@ e.g.:
 python deep_dream.py img/mypic.jpg results/dream
 ```
 
-It is preferrable to run this script on GPU, for speed.
+It is preferable to run this script on GPU, for speed.
 If running on CPU, prefer the TensorFlow backend (much faster).
 
 Example results: http://i.imgur.com/FX6ROg9.jpg
@@ -24,7 +24,7 @@ import h5py
 import os
 
 from keras.models import Sequential
-from keras.layers.convolutional import Convolution2D, ZeroPadding2D, MaxPooling2D
+from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
 from keras import backend as K
 
 parser = argparse.ArgumentParser(description='Deep Dreams with Keras.')
@@ -189,7 +189,7 @@ def eval_loss_and_grads(x):
 class Evaluator(object):
     def __init__(self):
         self.loss_value = None
-        self.grads_values = None
+        self.grad_values = None
 
     def loss(self, x):
         assert self.loss_value is None

examples/imdb_cnn.py

@@ -12,9 +12,9 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Lambda
-from keras.layers.embeddings import Embedding
-from keras.layers.convolutional import Convolution1D
+from keras.layers import Dense, Dropout, Activation, Lambda
+from keras.layers import Embedding
+from keras.layers import Convolution1D
 from keras.datasets import imdb
 from keras import backend as K
 

examples/imdb_cnn_lstm.py

@@ -9,10 +9,10 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.embeddings import Embedding
-from keras.layers.recurrent import LSTM, GRU, SimpleRNN
-from keras.layers.convolutional import Convolution1D, MaxPooling1D
+from keras.layers import Dense, Dropout, Activation
+from keras.layers import Embedding
+from keras.layers import LSTM, GRU, SimpleRNN
+from keras.layers import Convolution1D, MaxPooling1D
 from keras.datasets import imdb
 
 

examples/imdb_lstm.py

@@ -19,9 +19,8 @@ np.random.seed(1337)  # for reproducibility
 from keras.preprocessing import sequence
 from keras.utils import np_utils
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.embeddings import Embedding
-from keras.layers.recurrent import LSTM, SimpleRNN, GRU
+from keras.layers import Dense, Dropout, Activation, Embedding
+from keras.layers import LSTM, SimpleRNN, GRU
 from keras.datasets import imdb
 
 max_features = 20000

examples/inception_v3.py

@@ -0,0 +1,290 @@
+'''This script demonstrates how to build the Inception v3 architecture
+using the Keras functional API.
+We are not actually training it here, for lack of appropriate data.
+
+For more information about this architecture, see:
+
+"Rethinking the Inception Architecture for Computer Vision"
+Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, Zbigniew Wojna
+http://arxiv.org/abs/1512.00567
+'''
+from keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from keras.layers import BatchNormalization, Flatten, Dense, Dropout
+from keras.layers import Input, merge
+from keras.models import Model
+from keras import regularizers
+
+
+# global constants
+NB_CLASS = 1000  # number of classes
+DIM_ORDERING = 'th'  # 'th' (channels, width, height) or 'tf' (width, height, channels)
+WEIGHT_DECAY = 0.  # L2 regularization factor
+USE_BN = False  # whether to use batch normalization
+
+
+def conv2D_bn(x, nb_filter, nb_row, nb_col,
+              border_mode='same', subsample=(1, 1),
+              activation='relu', batch_norm=USE_BN,
+              weight_decay=WEIGHT_DECAY, dim_ordering=DIM_ORDERING):
+    '''Utility function to apply to a tensor a module conv + BN
+    with optional weight decay (L2 weight regularization).
+    '''
+    if weight_decay:
+        W_regularizer = regularizers.l2(weight_decay)
+        b_regularizer = regularizers.l2(weight_decay)
+    else:
+        W_regularizer = None
+        b_regularizer = None
+    x = Convolution2D(nb_filter, nb_row, nb_col,
+                      subsample=subsample,
+                      activation=activation,
+                      border_mode=border_mode,
+                      W_regularizer=W_regularizer,
+                      b_regularizer=b_regularizer,
+                      dim_ordering=dim_ordering)(x)
+    if batch_norm:
+        x = BatchNormalization()(x)
+    return x
+
+# Define image input layer
+
+if DIM_ORDERING == 'th':
+    img_input = Input(shape=(3, 299, 299))
+    CONCAT_AXIS = 1
+elif DIM_ORDERING == 'tf':
+    img_input = Input(shape=(299, 299, 3))
+    CONCAT_AXIS = 3
+else:
+    raise Exception('Invalid dim ordering: ' + str(DIM_ORDERING))
+
+# Entry module
+
+x = conv2D_bn(img_input, 32, 3, 3, subsample=(2, 2), border_mode='valid')
+x = conv2D_bn(x, 32, 3, 3, border_mode='valid')
+x = conv2D_bn(x, 64, 3, 3)
+x = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)
+
+x = conv2D_bn(x, 80, 1, 1, border_mode='valid')
+x = conv2D_bn(x, 192, 3, 3, border_mode='valid')
+x = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)
+
+# mixed: 35 x 35 x 256
+
+branch1x1 = conv2D_bn(x, 64, 1, 1)
+
+branch5x5 = conv2D_bn(x, 48, 1, 1)
+branch5x5 = conv2D_bn(branch5x5, 64, 5, 5)
+
+branch3x3dbl = conv2D_bn(x, 64, 1, 1)
+branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
+branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
+
+branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
+branch_pool = conv2D_bn(branch_pool, 32, 1, 1)
+x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# mixed_1: 35 x 35 x 288
+
+branch1x1 = conv2D_bn(x, 64, 1, 1)
+
+branch5x5 = conv2D_bn(x, 48, 1, 1)
+branch5x5 = conv2D_bn(branch5x5, 64, 5, 5)
+
+branch3x3dbl = conv2D_bn(x, 64, 1, 1)
+branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
+branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
+
+branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
+branch_pool = conv2D_bn(branch_pool, 64, 1, 1)
+x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# mixed2: 35 x 35 x 288
+
+branch1x1 = conv2D_bn(x, 64, 1, 1)
+
+branch5x5 = conv2D_bn(x, 48, 1, 1)
+branch5x5 = conv2D_bn(branch5x5, 64, 5, 5)
+
+branch3x3dbl = conv2D_bn(x, 64, 1, 1)
+branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
+branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
+
+branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
+branch_pool = conv2D_bn(branch_pool, 64, 1, 1)
+x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# mixed3: 17 x 17 x 768
+
+branch3x3 = conv2D_bn(x, 384, 3, 3, subsample=(2, 2), border_mode='valid')
+
+branch3x3dbl = conv2D_bn(x, 64, 1, 1)
+branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
+branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3, subsample=(2, 2), border_mode='valid')
+
+branch_pool = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)
+x = merge([branch3x3, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# mixed4: 17 x 17 x 768
+
+branch1x1 = conv2D_bn(x, 192, 1, 1)
+
+branch7x7 = conv2D_bn(x, 128, 1, 1)
+branch7x7 = conv2D_bn(branch7x7, 128, 1, 7)
+branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)
+
+branch7x7dbl = conv2D_bn(x, 128, 1, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 128, 7, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 128, 1, 7)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 128, 7, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
+
+branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
+branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
+x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# mixed5: 17 x 17 x 768
+
+branch1x1 = conv2D_bn(x, 192, 1, 1)
+
+branch7x7 = conv2D_bn(x, 160, 1, 1)
+branch7x7 = conv2D_bn(branch7x7, 160, 1, 7)
+branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)
+
+branch7x7dbl = conv2D_bn(x, 160, 1, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 1, 7)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
+
+branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
+branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
+x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# mixed5: 17 x 17 x 768
+
+branch1x1 = conv2D_bn(x, 192, 1, 1)
+
+branch7x7 = conv2D_bn(x, 160, 1, 1)
+branch7x7 = conv2D_bn(branch7x7, 160, 1, 7)
+branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)
+
+branch7x7dbl = conv2D_bn(x, 160, 1, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 1, 7)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
+
+branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
+branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
+x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# mixed6: 17 x 17 x 768
+
+branch1x1 = conv2D_bn(x, 192, 1, 1)
+
+branch7x7 = conv2D_bn(x, 160, 1, 1)
+branch7x7 = conv2D_bn(branch7x7, 160, 1, 7)
+branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)
+
+branch7x7dbl = conv2D_bn(x, 160, 1, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
+
+branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
+branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
+x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# mixed7: 17 x 17 x 768
+
+branch1x1 = conv2D_bn(x, 192, 1, 1)
+
+branch7x7 = conv2D_bn(x, 192, 1, 1)
+branch7x7 = conv2D_bn(branch7x7, 192, 1, 7)
+branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)
+
+branch7x7dbl = conv2D_bn(x, 160, 1, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 7, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 7, 1)
+branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
+
+branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
+branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
+x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# Auxiliary head
+
+aux_logits = AveragePooling2D((5, 5), strides=(3, 3), dim_ordering=DIM_ORDERING)(x)
+aux_logits = conv2D_bn(aux_logits, 128, 1, 1)
+aux_logits = conv2D_bn(aux_logits, 728, 5, 5, border_mode='valid')
+aux_logits = Flatten()(aux_logits)
+aux_preds = Dense(NB_CLASS, activation='softmax')(aux_logits)
+
+# mixed8: 8 x 8 x 1280
+
+branch3x3 = conv2D_bn(x, 192, 1, 1)
+branch3x3 = conv2D_bn(branch3x3, 320, 3, 3, subsample=(2, 2), border_mode='valid')
+
+branch7x7x3 = conv2D_bn(x, 192, 1, 1)
+branch7x7x3 = conv2D_bn(branch7x7x3, 192, 1, 7)
+branch7x7x3 = conv2D_bn(branch7x7x3, 192, 7, 1)
+branch7x7x3 = conv2D_bn(branch7x7x3, 192, 3, 3, subsample=(2, 2), border_mode='valid')
+
+branch_pool = AveragePooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)
+x = merge([branch3x3, branch7x7x3, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# mixed9: 8 x 8 x 2048
+
+branch1x1 = conv2D_bn(x, 320, 1, 1)
+
+branch3x3 = conv2D_bn(x, 384, 1, 1)
+branch3x3_1 = conv2D_bn(branch3x3, 384, 1, 3)
+branch3x3_2 = conv2D_bn(branch3x3, 384, 3, 1)
+branch3x3 = merge([branch3x3_1, branch3x3_2], mode='concat', concat_axis=CONCAT_AXIS)
+
+branch3x3dbl = conv2D_bn(x, 448, 1, 1)
+branch3x3dbl = conv2D_bn(branch3x3dbl, 384, 3, 3)
+branch3x3dbl_1 = conv2D_bn(branch3x3dbl, 384, 1, 3)
+branch3x3dbl_2 = conv2D_bn(branch3x3dbl, 384, 3, 1)
+branch3x3dbl = merge([branch3x3dbl_1, branch3x3dbl_2], mode='concat', concat_axis=CONCAT_AXIS)
+
+branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
+branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
+x = merge([branch1x1, branch3x3, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# mixed10: 8 x 8 x 2048
+
+branch1x1 = conv2D_bn(x, 320, 1, 1)
+
+branch3x3 = conv2D_bn(x, 384, 1, 1)
+branch3x3_1 = conv2D_bn(branch3x3, 384, 1, 3)
+branch3x3_2 = conv2D_bn(branch3x3, 384, 3, 1)
+branch3x3 = merge([branch3x3_1, branch3x3_2], mode='concat', concat_axis=CONCAT_AXIS)
+
+branch3x3dbl = conv2D_bn(x, 448, 1, 1)
+branch3x3dbl = conv2D_bn(branch3x3dbl, 384, 3, 3)
+branch3x3dbl_1 = conv2D_bn(branch3x3dbl, 384, 1, 3)
+branch3x3dbl_2 = conv2D_bn(branch3x3dbl, 384, 3, 1)
+branch3x3dbl = merge([branch3x3dbl_1, branch3x3dbl_2], mode='concat', concat_axis=CONCAT_AXIS)
+
+branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
+branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
+x = merge([branch1x1, branch3x3, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
+
+# Final pooling and prediction
+
+x = AveragePooling2D((8, 8), strides=(1, 1), dim_ordering=DIM_ORDERING)(x)
+x = Dropout(0.5)(x)
+x = Flatten()(x)
+preds = Dense(NB_CLASS, activation='softmax')(x)
+
+# Define model
+
+model = Model(input=img_input, output=[preds, aux_preds])
+model.compile('rmsprop', 'categorical_crossentropy')
+
+# train via e.g. `model.fit(x_train, [y_train] * 2, batch_size=32, nb_epoch=100)`
+# Note that for a large dataset it would be preferable
+# to train using `fit_generator` (see Keras docs).

examples/lstm_benchmark.py

@@ -0,0 +1,83 @@
+'''Compare LSTM implementations on the IMDB sentiment classification task.
+
+consume_less='cpu' preprocesses input to the LSTM which typically results in
+faster computations at the expense of increased peak memory usage as the
+preprocessed input must be kept in memory.
+
+consume_less='mem' does away with the preprocessing, meaning that it might take
+a little longer, but should require less peak memory.
+
+consume_less='gpu' concatenates the input, output and forget gate's weights
+into one, large matrix, resulting in faster computation time as the GPU can
+utilize more cores, at the expense of reduced regularization because the same
+dropout is shared across the gates.
+
+Note that the relative performance of the different `consume_less` modes
+can vary depending on your device, your model and the size of your data.
+'''
+
+import time
+import numpy as np
+import matplotlib.pyplot as plt
+
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.layers import Embedding, Dense, LSTM
+from keras.datasets import imdb
+
+max_features = 20000
+max_length = 80
+embedding_dim = 256
+batch_size = 128
+epochs = 10
+modes = ['cpu', 'mem', 'gpu']
+
+print('Loading data...')
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
+X_train = sequence.pad_sequences(X_train, max_length)
+X_test = sequence.pad_sequences(X_test, max_length)
+
+# Compile and train different models while meauring performance.
+results = []
+for mode in modes:
+    print('Testing mode: consume_less="{}"'.format(mode))
+
+    model = Sequential()
+    model.add(Embedding(max_features, embedding_dim, input_length=max_length, dropout=0.2))
+    model.add(LSTM(embedding_dim, dropout_W=0.2, dropout_U=0.2, consume_less=mode))
+    model.add(Dense(1, activation='sigmoid'))
+    model.compile(loss='binary_crossentropy',
+                  optimizer='adam',
+                  metrics=['accuracy'])
+
+    start_time = time.time()
+    history = model.fit(X_train, y_train,
+                        batch_size=batch_size,
+                        nb_epoch=epochs,
+                        validation_data=(X_test, y_test))
+    average_time_per_epoch = (time.time() - start_time) / epochs
+
+    results.append((history, average_time_per_epoch))
+
+# Compare models' accuracy, loss and elapsed time per epoch.
+plt.style.use('ggplot')
+ax1 = plt.subplot2grid((2, 2), (0, 0))
+ax1.set_title('Accuracy')
+ax1.set_ylabel('Validation Accuracy')
+ax1.set_xlabel('Epochs')
+ax2 = plt.subplot2grid((2, 2), (1, 0))
+ax2.set_title('Loss')
+ax2.set_ylabel('Validation Loss')
+ax2.set_xlabel('Epochs')
+ax3 = plt.subplot2grid((2, 2), (0, 1), rowspan=2)
+ax3.set_title('Time')
+ax3.set_ylabel('Seconds')
+for mode, result in zip(modes, results):
+    ax1.plot(result[0].epoch, result[0].history['val_acc'], label=mode)
+    ax2.plot(result[0].epoch, result[0].history['val_loss'], label=mode)
+ax1.legend()
+ax2.legend()
+ax3.bar(np.arange(len(results)), [x[1] for x in results],
+        tick_label=modes, align='center')
+plt.tight_layout()
+plt.show()

examples/lstm_text_generation.py

@@ -12,8 +12,8 @@ has at least ~100k characters. ~1M is better.
 
 from __future__ import print_function
 from keras.models import Sequential
-from keras.layers.core import Dense, Activation, Dropout
-from keras.layers.recurrent import LSTM
+from keras.layers import Dense, Activation, Dropout
+from keras.layers import LSTM
 from keras.utils.data_utils import get_file
 import numpy as np
 import random
@@ -23,7 +23,7 @@ path = get_file('nietzsche.txt', origin="https://s3.amazonaws.com/text-datasets/
 text = open(path).read().lower()
 print('corpus length:', len(text))
 
-chars = set(text)
+chars = sorted(list(set(text)))
 print('total chars:', len(chars))
 char_indices = dict((c, i) for i, c in enumerate(chars))
 indices_char = dict((i, c) for i, c in enumerate(chars))
@@ -51,7 +51,6 @@ for i, sentence in enumerate(sentences):
 print('Build model...')
 model = Sequential()
 model.add(LSTM(512, return_sequences=True, input_shape=(maxlen, len(chars))))
-model.add(Dropout(0.2))
 model.add(LSTM(512, return_sequences=False))
 model.add(Dropout(0.2))
 model.add(Dense(len(chars)))

examples/mnist_cnn.py

@@ -11,8 +11,8 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.datasets import mnist
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Flatten
-from keras.layers.convolutional import Convolution2D, MaxPooling2D
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Convolution2D, MaxPooling2D
 from keras.utils import np_utils
 
 batch_size = 128

examples/mnist_irnn.py

@@ -3,7 +3,7 @@ with pixel-by-pixel sequential MNIST in
 "A Simple Way to Initialize Recurrent Networks of Rectified Linear Units"
 by Quoc V. Le, Navdeep Jaitly, Geoffrey E. Hinton
 
-arXiv:1504.00941v2 [cs.NE] 7 Apr 201
+arXiv:1504.00941v2 [cs.NE] 7 Apr 2015
 http://arxiv.org/pdf/1504.00941v2.pdf
 
 Optimizer is replaced with RMSprop which yields more stable and steady
@@ -17,9 +17,9 @@ from __future__ import print_function
 
 from keras.datasets import mnist
 from keras.models import Sequential
-from keras.layers.core import Dense, Activation
+from keras.layers import Dense, Activation
+from keras.layers import SimpleRNN
 from keras.initializations import normal, identity
-from keras.layers.recurrent import SimpleRNN
 from keras.optimizers import RMSprop
 from keras.utils import np_utils
 

examples/mnist_siamese_graph.py

@@ -30,7 +30,7 @@ def euclidean_distance(vects):
 
 def eucl_dist_output_shape(shapes):
     shape1, shape2 = shapes
-    return shape1
+    return (shape1[0], 1)
 
 
 def contrastive_loss(y_true, y_pred):

examples/mnist_sklearn_wrapper.py

@@ -0,0 +1,94 @@
+'''Example of how to use sklearn wrapper
+
+Builds simple CNN models on MNIST and uses sklearn's GridSearchCV to find best model
+'''
+
+from __future__ import print_function
+import numpy as np
+np.random.seed(1337)  # for reproducibility
+
+from keras.datasets import mnist
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Convolution2D, MaxPooling2D
+from keras.utils import np_utils
+from keras.wrappers.scikit_learn import KerasClassifier
+from sklearn.grid_search import GridSearchCV
+
+
+nb_classes = 10
+
+# input image dimensions
+img_rows, img_cols = 28, 28
+
+# load training data and do basic data normalization
+(X_train, y_train), (X_test, y_test) = mnist.load_data()
+X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
+X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
+X_train = X_train.astype('float32')
+X_test = X_test.astype('float32')
+X_train /= 255
+X_test /= 255
+
+# convert class vectors to binary class matrices
+y_train = np_utils.to_categorical(y_train, nb_classes)
+y_test = np_utils.to_categorical(y_test, nb_classes)
+
+def make_model(dense_layer_sizes, nb_filters, nb_conv, nb_pool):
+    '''Creates model comprised of 2 convolutional layers followed by dense layers
+
+    dense_layer_sizes: List of layer sizes. This list has one number for each layer
+    nb_filters: Number of convolutional filters in each convolutional layer
+    nb_conv: Convolutional kernel size
+    nb_pool: Size of pooling area for max pooling
+    '''
+
+    model = Sequential()
+
+    model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
+                            border_mode='valid',
+                            input_shape=(1, img_rows, img_cols)))
+    model.add(Activation('relu'))
+    model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
+    model.add(Dropout(0.25))
+
+    model.add(Flatten())
+    for layer_size in dense_layer_sizes:
+        model.add(Dense(layer_size))
+    model.add(Activation('relu'))
+    model.add(Dropout(0.5))
+    model.add(Dense(nb_classes))
+    model.add(Activation('softmax'))
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer='adadelta',
+                  metrics=['accuracy'])
+
+    return model
+
+dense_size_candidates = [[32], [64], [32, 32], [64, 64]]
+my_classifier = KerasClassifier(make_model, batch_size=32)
+validator = GridSearchCV(my_classifier,
+                         param_grid={'dense_layer_sizes': dense_size_candidates,
+                                     # nb_epoch is avail for tuning even when not
+                                     # an argument to model building function
+                                     'nb_epoch': [3, 6],
+                                     'nb_filters': [8],
+                                     'nb_conv': [3],
+                                     'nb_pool': [2]},
+                         scoring='log_loss',
+                         n_jobs=1)
+validator.fit(X_train, y_train)
+
+print('The parameters of the best model are: ')
+print(validator.best_params_)
+
+# validator.best_estimator_ returns sklearn-wrapped version of best model.
+# validator.best_estimator_.model returns the (unwrapped) keras model
+best_model = validator.best_estimator_.model
+metric_names = best_model.metrics_names
+metric_values = best_model.evaluate(X_test, y_test)
+for metric, value in zip(metric_names, metric_values):
+    print(metric, ': ', value)

examples/mnist_transfer_cnn.py

@@ -19,8 +19,8 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.datasets import mnist
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Flatten
-from keras.layers.convolutional import Convolution2D, MaxPooling2D
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Convolution2D, MaxPooling2D
 from keras.utils import np_utils
 
 

examples/neural_style_transfer.py

@@ -14,7 +14,7 @@ e.g.:
 python neural_style_transfer.py img/tuebingen.jpg img/starry_night.jpg results/my_result
 ```
 
-It is preferrable to run this script on GPU, for speed.
+It is preferable to run this script on GPU, for speed.
 If running on CPU, prefer the TensorFlow backend (much faster).
 
 Example result: https://twitter.com/fchollet/status/686631033085677568
@@ -34,7 +34,7 @@ the pixels of the combination image, giving it visual coherence.
 
 - The style loss is where the deep learning keeps in --that one is defined
 using a deep convolutional neural network. Precisely, it consists in a sum of
-L2 distances betwen the Gram matrices of the representations of
+L2 distances between the Gram matrices of the representations of
 the base image and the style reference image, extracted from
 different layers of a convnet (trained on ImageNet). The general idea
 is to capture color/texture information at different spatial
@@ -58,7 +58,7 @@ import argparse
 import h5py
 
 from keras.models import Sequential
-from keras.layers.convolutional import Convolution2D, ZeroPadding2D, MaxPooling2D
+from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
 from keras import backend as K
 
 parser = argparse.ArgumentParser(description='Neural style transfer with Keras.')

examples/reuters_mlp.py

@@ -8,8 +8,7 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.datasets import reuters
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.normalization import BatchNormalization
+from keras.layers import Dense, Dropout, Activation
 from keras.utils import np_utils
 from keras.preprocessing.text import Tokenizer
 

examples/stateful_lstm.py

@@ -5,8 +5,7 @@ from __future__ import print_function
 import numpy as np
 import matplotlib.pyplot as plt
 from keras.models import Sequential
-from keras.layers.core import Dense
-from keras.layers.recurrent import LSTM
+from keras.layers import Dense, LSTM
 
 
 # since we are using stateful rnn tsteps can be set to 1
@@ -75,7 +74,7 @@ for i in range(epochs):
 print('Predicting')
 predicted_output = model.predict(cos, batch_size=batch_size)
 
-print('Ploting Results')
+print('Plotting Results')
 plt.subplot(2, 1, 1)
 plt.plot(expected_output)
 plt.title('Expected')

examples/variational_autoencoder.py

@@ -0,0 +1,98 @@
+'''This script demonstrates how to build a variational autoencoder with Keras.
+
+Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
+'''
+import numpy as np
+import matplotlib.pyplot as plt
+
+from keras.layers import Input, Dense, Lambda
+from keras.models import Model
+from keras import backend as K
+from keras import objectives
+from keras.datasets import mnist
+
+batch_size = 16
+original_dim = 784
+latent_dim = 2
+intermediate_dim = 128
+epsilon_std = 0.01
+nb_epoch = 40
+
+x = Input(batch_shape=(batch_size, original_dim))
+h = Dense(intermediate_dim, activation='relu')(x)
+z_mean = Dense(latent_dim)(h)
+z_log_std = Dense(latent_dim)(h)
+
+def sampling(args):
+    z_mean, z_log_std = args
+    epsilon = K.random_normal(shape=(batch_size, latent_dim),
+                              mean=0., std=epsilon_std)
+    return z_mean + K.exp(z_log_std) * epsilon
+
+# note that "output_shape" isn't necessary with the TensorFlow backend
+# so you could write `Lambda(sampling)([z_mean, z_log_std])`
+z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_std])
+
+# we instantiate these layers separately so as to reuse them later
+decoder_h = Dense(intermediate_dim, activation='relu')
+decoder_mean = Dense(original_dim, activation='sigmoid')
+h_decoded = decoder_h(z)
+x_decoded_mean = decoder_mean(h_decoded)
+
+def vae_loss(x, x_decoded_mean):
+    xent_loss = objectives.binary_crossentropy(x, x_decoded_mean)
+    kl_loss = - 0.5 * K.mean(1 + z_log_std - K.square(z_mean) - K.exp(z_log_std), axis=-1)
+    return xent_loss + kl_loss
+
+vae = Model(x, x_decoded_mean)
+vae.compile(optimizer='rmsprop', loss=vae_loss)
+
+# train the VAE on MNIST digits
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+x_train = x_train.astype('float32') / 255.
+x_test = x_test.astype('float32') / 255.
+x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
+x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))
+
+vae.fit(x_train, x_train,
+        shuffle=True,
+        nb_epoch=nb_epoch,
+        batch_size=batch_size,
+        validation_data=(x_test, x_test))
+
+# build a model to project inputs on the latent space
+encoder = Model(x, z_mean)
+
+# display a 2D plot of the digit classes in the latent space
+x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
+plt.figure(figsize=(6, 6))
+plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
+plt.colorbar()
+plt.show()
+
+# build a digit generator that can sample from the learned distribution
+decoder_input = Input(shape=(latent_dim,))
+_h_decoded = decoder_h(decoder_input)
+_x_decoded_mean = decoder_mean(_h_decoded)
+generator = Model(decoder_input, _x_decoded_mean)
+
+# display a 2D manifold of the digits
+n = 15  # figure with 15x15 digits
+digit_size = 28
+figure = np.zeros((digit_size * n, digit_size * n))
+# we will sample n points within [-15, 15] standard deviations
+grid_x = np.linspace(-15, 15, n)
+grid_y = np.linspace(-15, 15, n)
+
+for i, yi in enumerate(grid_x):
+    for j, xi in enumerate(grid_y):
+        z_sample = np.array([[xi, yi]]) * epsilon_std
+        x_decoded = generator.predict(z_sample)
+        digit = x_decoded[0].reshape(digit_size, digit_size)
+        figure[i * digit_size: (i + 1) * digit_size,
+               j * digit_size: (j + 1) * digit_size] = digit
+
+plt.figure(figsize=(10, 10))
+plt.imshow(figure)
+plt.show()

keras/__init__.py

@@ -1 +1,18 @@
-__version__ = '1.0.1'
+from __future__ import absolute_import
+from . import backend
+from . import datasets
+from . import engine
+from . import layers
+from . import preprocessing
+from . import utils
+from . import wrappers
+from . import callbacks
+from . import constraints
+from . import initializations
+from . import metrics
+from . import models
+from . import objectives
+from . import optimizers
+from . import regularizers
+
+__version__ = '1.0.5'

keras/activations.py

@@ -19,6 +19,10 @@ def softplus(x):
     return K.softplus(x)
 
 
+def softsign(x):
+    return K.softsign(x)
+
+
 def relu(x, alpha=0., max_value=None):
     return K.relu(x, alpha=alpha, max_value=max_value)
 

keras/backend/__init__.py

@@ -9,6 +9,8 @@ from .common import set_epsilon
 from .common import set_floatx
 from .common import get_uid
 from .common import cast_to_floatx
+from .common import image_dim_ordering
+from .common import set_image_dim_ordering
 
 _keras_base_dir = os.path.expanduser('~')
 if not os.access(_keras_base_dir, os.W_OK):
@@ -28,24 +30,27 @@ if os.path.exists(_config_path):
     assert type(_epsilon) == float
     _backend = _config.get('backend', _BACKEND)
     assert _backend in {'theano', 'tensorflow'}
+    _image_dim_ordering = _config.get('image_dim_ordering', image_dim_ordering())
+    assert _image_dim_ordering in {'tf', 'th'}
 
     set_floatx(_floatx)
     set_epsilon(_epsilon)
     _BACKEND = _backend
-else:
-    # save config file, for easy edition
-    _config = {'floatx': floatx(),
-               'epsilon': epsilon(),
-               'backend': _BACKEND}
-    with open(_config_path, 'w') as f:
-        # add new line in order for bash 'cat' display the content correctly
-        f.write(json.dumps(_config) + '\n')
+
+# save config file
+_config = {'floatx': floatx(),
+           'epsilon': epsilon(),
+           'backend': _BACKEND,
+           'image_dim_ordering': image_dim_ordering()}
+with open(_config_path, 'w') as f:
+    f.write(json.dumps(_config, indent=4))
 
 if 'KERAS_BACKEND' in os.environ:
     _backend = os.environ['KERAS_BACKEND']
     assert _backend in {'theano', 'tensorflow'}
     _BACKEND = _backend
 
+# import backend
 if _BACKEND == 'theano':
     sys.stderr.write('Using Theano backend.\n')
     from .theano_backend import *

keras/backend/common.py

@@ -4,13 +4,20 @@ import numpy as np
 _FLOATX = 'float32'
 _EPSILON = 10e-8
 _UID_PREFIXES = {}
+_IMAGE_DIM_ORDERING = 'th'
 
 
 def epsilon():
+    '''Returns the value of the fuzz
+    factor used in numeric expressions.
+    '''
     return _EPSILON
 
 
 def set_epsilon(e):
+    '''Sets the value of the fuzz
+    factor used in numeric expressions.
+    '''
     global _EPSILON
     _EPSILON = e
 
@@ -26,8 +33,7 @@ def set_floatx(floatx):
     global _FLOATX
     if floatx not in {'float16', 'float32', 'float64'}:
         raise Exception('Unknown floatx type: ' + str(floatx))
-    floatx = str(floatx)
-    _FLOATX = floatx
+    _FLOATX = str(floatx)
 
 
 def cast_to_floatx(x):
@@ -36,6 +42,23 @@ def cast_to_floatx(x):
     return np.asarray(x, dtype=_FLOATX)
 
 
+def image_dim_ordering():
+    '''Returns the image dimension ordering
+    convention ('th' or 'tf').
+    '''
+    return _IMAGE_DIM_ORDERING
+
+
+def set_image_dim_ordering(dim_ordering):
+    '''Sets the value of the image dimension
+    ordering convention ('th' or 'tf').
+    '''
+    global _IMAGE_DIM_ORDERING
+    if dim_ordering not in {'tf', 'th'}:
+        raise Exception('Unknown dim_ordering:', dim_ordering)
+    _IMAGE_DIM_ORDERING = str(dim_ordering)
+
+
 def get_uid(prefix=''):
     if prefix not in _UID_PREFIXES:
         _UID_PREFIXES[prefix] = 1

keras/backend/tensorflow_backend.py

@@ -27,20 +27,33 @@ def set_learning_phase(value):
 
 
 def get_session():
-    '''Returns the TF session in use by the backend.
+    '''Returns the TF session to be used by the backend.
+
+    If a default TensorFlow session is available, we will return it.
+
+    Else, we will return the global Keras session.
+
+    If no global Keras session exists at this point:
+    we will create a new global session.
+
+    Note that you can manually set the global session
+    via `K.set_session(sess)`.
     '''
     global _SESSION
+    if tf.get_default_session() is not None:
+        return tf.get_default_session()
     if _SESSION is None:
         if not os.environ.get('OMP_NUM_THREADS'):
             _SESSION = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
         else:
             nb_thread = int(os.environ.get('OMP_NUM_THREADS'))
-            _SESSION = tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=nb_thread, allow_soft_placement=True))
+            _SESSION = tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=nb_thread,
+                                                        allow_soft_placement=True))
     return _SESSION
 
 
 def set_session(session):
-    '''Sets the TF session.
+    '''Sets the global TF session.
     '''
     global _SESSION
     _SESSION = session
@@ -60,7 +73,20 @@ def variable(value, dtype=_FLOATX, name=None):
         Tensor variable instance.
     '''
     v = tf.Variable(np.asarray(value, dtype=dtype), name=name)
-    get_session().run(v.initializer)
+    if tf.get_default_graph() is get_session().graph:
+        try:
+            get_session().run(v.initializer)
+        except tf.errors.InvalidArgumentError:
+            warnings.warn('Could not automatically initialize variable, '
+                          'make sure you do it manually (e.g. via '
+                          '`tf.initialize_all_variables()`).')
+    else:
+        warnings.warn('The default TensorFlow graph is not the graph '
+                      'associated with the TensorFlow session currently '
+                      'registered with Keras, and as such Keras '
+                      'was not able to automatically initialize a variable. '
+                      'You should consider registering the proper session '
+                      'with Keras via `K.set_session(sess)`.')
     return v
 
 
@@ -97,6 +123,7 @@ def shape(x):
 def int_shape(x):
     '''Returns the shape of a tensor as a tuple of
     integers or None entries.
+    Note that this function only works with TensorFlow.
     '''
     shape = x.get_shape()
     return tuple([i.__int__() for i in shape])
@@ -136,6 +163,12 @@ def ones(shape, dtype=_FLOATX, name=None):
     return variable(np.ones(shape), dtype, name)
 
 
+def eye(size, dtype=_FLOATX, name=None):
+    '''Instantiate an identity matrix.
+    '''
+    return variable(np.eye(size), dtype, name)
+
+
 def zeros_like(x, name=None):
     '''Instantiates an all-zeros tensor
     of the same shape as another tensor.
@@ -282,21 +315,27 @@ def prod(x, axis=None, keepdims=False):
     return tf.reduce_prod(x, reduction_indices=axis, keep_dims=keepdims)
 
 
-def std(x, axis=None, keepdims=False):
-    '''Standard deviation of a tensor, alongside the specificied axis.
+def var(x, axis=None, keepdims=False):
+    '''Variance of a tensor, alongside the specified axis.
     '''
     axis = _normalize_axis(axis, ndim(x))
     if x.dtype.base_dtype == tf.bool:
         x = tf.cast(x, _FLOATX)
     m = tf.reduce_mean(x, reduction_indices=axis, keep_dims=True)
     devs_squared = tf.square(x - m)
-    return tf.sqrt(tf.reduce_mean(devs_squared,
-                                  reduction_indices=axis,
-                                  keep_dims=keepdims))
+    return tf.reduce_mean(devs_squared,
+                          reduction_indices=axis,
+                          keep_dims=keepdims)
+
+
+def std(x, axis=None, keepdims=False):
+    '''Standard deviation of a tensor, alongside the specified axis.
+    '''
+    return tf.sqrt(var(x, axis=axis, keepdims=keepdims))
 
 
 def mean(x, axis=None, keepdims=False):
-    '''Mean of a tensor, alongside the specificied axis.
+    '''Mean of a tensor, alongside the specified axis.
     '''
     axis = _normalize_axis(axis, ndim(x))
     if x.dtype.base_dtype == tf.bool:
@@ -315,6 +354,17 @@ def any(x, axis=None, keepdims=False):
     return tf.cast(x, tf.uint8)
 
 
+def all(x, axis=None, keepdims=False):
+    '''Bitwise reduction (logical AND).
+
+    Returns an uint8 tensor
+    '''
+    axis = _normalize_axis(axis, ndim(x))
+    x = tf.cast(x, tf.bool)
+    x = tf.reduce_all(x, reduction_indices=axis, keep_dims=keepdims)
+    return tf.cast(x, tf.uint8)
+
+
 def argmax(x, axis=-1):
     '''Returns the index of the maximum value
     along a tensor axis.
@@ -418,6 +468,18 @@ def minimum(x, y):
     return tf.minimum(x, y)
 
 
+def sin(x):
+    '''Computes sin of x element-wise.
+    '''
+    return tf.sin(x)
+
+
+def cos(x):
+    '''Computes cos of x element-wise.
+    '''
+    return tf.cos(x)
+
+
 # SHAPE OPERATIONS
 
 def concatenate(tensors, axis=-1):
@@ -455,15 +517,21 @@ def resize_images(X, height_factor, width_factor, dim_ordering):
     positive integers.
     '''
     if dim_ordering == 'th':
+        original_shape = int_shape(X)
         new_shape = tf.shape(X)[2:]
         new_shape *= tf.constant(np.array([height_factor, width_factor]).astype('int32'))
         X = permute_dimensions(X, [0, 2, 3, 1])
         X = tf.image.resize_nearest_neighbor(X, new_shape)
-        return permute_dimensions(X, [0, 3, 1, 2])
+        X = permute_dimensions(X, [0, 3, 1, 2])
+        X.set_shape((None, None, original_shape[2] * height_factor, original_shape[3] * width_factor))
+        return X
     elif dim_ordering == 'tf':
+        original_shape = int_shape(X)
         new_shape = tf.shape(X)[1:3]
         new_shape *= tf.constant(np.array([height_factor, width_factor]).astype('int32'))
-        return tf.image.resize_nearest_neighbor(X, new_shape)
+        X = tf.image.resize_nearest_neighbor(X, new_shape)
+        X.set_shape((None, original_shape[1] * height_factor, original_shape[2] * width_factor, None))
+        return X
     else:
         raise Exception('Invalid dim_ordering: ' + dim_ordering)
 
@@ -495,6 +563,8 @@ def repeat(x, n):
 
 
 def tile(x, n):
+    if not hasattr(n, 'shape') and not hasattr(n, '__len__'):
+        n = [n]
     return tf.tile(x, n)
 
 
@@ -558,6 +628,16 @@ def get_value(x):
     return x.eval(session=get_session())
 
 
+def batch_get_value(xs):
+    '''Returns the value of more than one tensor variable,
+    as a list of Numpy arrays.
+    '''
+    if xs:
+        return get_session().run(xs)
+    else:
+        return []
+
+
 def set_value(x, value):
     '''Sets the value of a tensor variable,
     from a Numpy array.
@@ -808,6 +888,10 @@ def softplus(x):
     return tf.nn.softplus(x)
 
 
+def softsign(x):
+    return tf.nn.softsign(x)
+
+
 def categorical_crossentropy(output, target, from_logits=False):
     '''Categorical crossentropy between an output tensor
     and a target tensor, where the target is a tensor of the same
@@ -905,7 +989,7 @@ def dropout(x, level, seed=None):
 
 
 def l2_normalize(x, axis):
-    '''Normalizes a tensor wrt the L2 norm alonside the specified axis.
+    '''Normalizes a tensor wrt the L2 norm alongside the specified axis.
     '''
     if axis < 0:
         axis = axis % len(x.get_shape())

keras/backend/theano_backend.py

@@ -3,6 +3,10 @@ from theano import tensor as T
 from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
 from theano.tensor.signal import pool
 from theano.tensor.nnet import conv3d2d
+try:
+    from theano.tensor.nnet.nnet import softsign as T_softsign
+except ImportError:
+    from theano.sandbox.softsign import softsign as T_softsign
 import inspect
 import numpy as np
 from .common import _FLOATX, _EPSILON
@@ -79,6 +83,12 @@ def ones(shape, dtype=_FLOATX, name=None):
     return variable(np.ones(shape), dtype, name)
 
 
+def eye(size, dtype=_FLOATX, name=None):
+    '''Instantiate an identity matrix.
+    '''
+    return variable(np.eye(size), dtype, name)
+
+
 def ones_like(x):
     return T.ones_like(x)
 
@@ -190,12 +200,22 @@ def std(x, axis=None, keepdims=False):
     return T.std(x, axis=axis, keepdims=keepdims)
 
 
+def var(x, axis=None, keepdims=False):
+    return T.var(x, axis=axis, keepdims=keepdims)
+
+
 def any(x, axis=None, keepdims=False):
     '''Bitwise reduction (logical OR).
     '''
     return T.any(x, axis=axis, keepdims=keepdims)
 
 
+def all(x, axis=None, keepdims=False):
+    '''Bitwise reduction (logical AND).
+    '''
+    return T.all(x, axis=axis, keepdims=keepdims)
+
+
 def argmax(x, axis=-1):
     return T.argmax(x, axis=axis, keepdims=False)
 
@@ -259,6 +279,14 @@ def minimum(x, y):
     return T.minimum(x, y)
 
 
+def sin(x):
+    return T.sin(x)
+
+
+def cos(x):
+    return T.cos(x)
+
+
 # SHAPE OPERATIONS
 
 def concatenate(tensors, axis=-1):
@@ -371,15 +399,18 @@ def expand_dims(x, dim=-1):
 def squeeze(x, axis):
     '''Remove a 1-dimension from the tensor at index "axis".
     '''
-    x = T.addbroadcast(x, axis)
-    return T.squeeze(x)
+    broadcastable = x.broadcastable[:axis] + x.broadcastable[axis+1:]
+    x = T.patternbroadcast(x, [i == axis for i in range(x.type.ndim)])
+    x = T.squeeze(x)
+    x = T.patternbroadcast(x, broadcastable)
+    return x
 
 
 def temporal_padding(x, padding=1):
     '''Pad the middle dimension of a 3D tensor
     with "padding" zeros left and right.
 
-    Appologies for the inane API, but Theano makes this
+    Apologies for the inane API, but Theano makes this
     really hard.
     '''
     input_shape = x.shape
@@ -469,6 +500,13 @@ def get_value(x):
     return x.get_value()
 
 
+def batch_get_value(xs):
+    '''Returns the value of more than one tensor variable,
+    as a list of Numpy arrays.
+    '''
+    return [get_value(x) for x in xs]
+
+
 def set_value(x, value):
     x.set_value(np.asarray(value, dtype=x.dtype))
 
@@ -558,15 +596,15 @@ def rnn(step_function, inputs, initial_states,
     axes = [1, 0] + list(range(2, ndim))
     inputs = inputs.dimshuffle(axes)
 
+    if constants is None:
+        constants = []
+
     if mask is not None:
         if mask.ndim == ndim-1:
             mask = expand_dims(mask)
         assert mask.ndim == ndim
         mask = mask.dimshuffle(axes)
 
-        if constants is None:
-            constants = []
-
         if unroll:
             indices = list(range(input_length))
             if go_backwards:
@@ -576,7 +614,7 @@ def rnn(step_function, inputs, initial_states,
             successive_states = []
             states = initial_states
             for i in indices:
-                output, new_states = step_function(inputs[i], states)
+                output, new_states = step_function(inputs[i], states + constants)
 
                 if len(successive_outputs) == 0:
                     prev_output = zeros_like(output)
@@ -635,7 +673,7 @@ def rnn(step_function, inputs, initial_states,
             successive_states = []
             states = initial_states
             for i in indices:
-                output, states = step_function(inputs[i], states)
+                output, states = step_function(inputs[i], states + constants)
                 successive_outputs.append(output)
                 successive_states.append(states)
             outputs = T.stack(*successive_outputs)
@@ -711,6 +749,10 @@ def softplus(x):
     return T.nnet.softplus(x)
 
 
+def softsign(x):
+    return T_softsign(x)
+
+
 def categorical_crossentropy(output, target, from_logits=False):
     if from_logits:
         output = T.nnet.softmax(output)
@@ -753,7 +795,7 @@ def dropout(x, level, seed=None):
     if level < 0. or level >= 1:
         raise Exception('Dropout level must be in interval [0, 1[.')
     if seed is None:
-        seed = np.random.randint(10e6)
+        seed = np.random.randint(1, 10e6)
     rng = RandomStreams(seed=seed)
     retain_prob = 1. - level
     x *= rng.binomial(x.shape, p=retain_prob, dtype=x.dtype)
@@ -998,27 +1040,20 @@ def pool3d(x, pool_size, strides=(1, 1, 1), border_mode='valid',
 
 def random_normal(shape, mean=0.0, std=1.0, dtype=_FLOATX, seed=None):
     if seed is None:
-        seed = np.random.randint(10e6)
+        seed = np.random.randint(1, 10e6)
     rng = RandomStreams(seed=seed)
     return rng.normal(size=shape, avg=mean, std=std, dtype=dtype)
 
 
 def random_uniform(shape, low=0.0, high=1.0, dtype=_FLOATX, seed=None):
     if seed is None:
-        seed = np.random.randint(10e6)
+        seed = np.random.randint(1, 10e6)
     rng = RandomStreams(seed=seed)
     return rng.uniform(shape, low=low, high=high, dtype=dtype)
 
 
 def random_binomial(shape, p=0.0, dtype=_FLOATX, seed=None):
     if seed is None:
-        seed = np.random.randint(10e6)
+        seed = np.random.randint(1, 10e6)
     rng = RandomStreams(seed=seed)
     return rng.binomial(shape, p=p, dtype=dtype)
-
-'''
-more TODO:
-
-tensordot -> soon to be introduced in TF
-batched_tensordot -> reimplement
-'''

keras/callbacks.py

@@ -192,7 +192,7 @@ class ProgbarLogger(Callback):
             if k in logs:
                 self.log_values.append((k, logs[k]))
         if self.verbose:
-            self.progbar.update(self.seen, self.log_values)
+            self.progbar.update(self.seen, self.log_values, force=True)
 
 
 class History(Callback):
@@ -210,10 +210,7 @@ class History(Callback):
     def on_epoch_end(self, epoch, logs={}):
         self.epoch.append(epoch)
         for k, v in logs.items():
-            if k not in self.history:
-                self.history[k] = []
-            self.history[k].append(v)
-
+            self.history.setdefault(k, []).append(v)
 
 class ModelCheckpoint(Callback):
     '''Save the model after every epoch.
@@ -246,7 +243,7 @@ class ModelCheckpoint(Callback):
     def __init__(self, filepath, monitor='val_loss', verbose=0,
                  save_best_only=False, mode='auto'):
 
-        super(Callback, self).__init__()
+        super(ModelCheckpoint, self).__init__()
         self.monitor = monitor
         self.verbose = verbose
         self.filepath = filepath
@@ -313,7 +310,7 @@ class EarlyStopping(Callback):
             monitored has stopped increasing.
     '''
     def __init__(self, monitor='val_loss', patience=0, verbose=0, mode='auto'):
-        super(Callback, self).__init__()
+        super(EarlyStopping, self).__init__()
 
         self.monitor = monitor
         self.patience = patience
@@ -327,17 +324,17 @@ class EarlyStopping(Callback):
 
         if mode == 'min':
             self.monitor_op = np.less
-            self.best = np.Inf
         elif mode == 'max':
             self.monitor_op = np.greater
-            self.best = -np.Inf
         else:
             if 'acc' in self.monitor:
                 self.monitor_op = np.greater
-                self.best = -np.Inf
             else:
                 self.monitor_op = np.less
-                self.best = np.Inf
+
+    def on_train_begin(self, logs={}):
+        self.wait = 0       # Allow instances to be re-used
+        self.best = np.Inf if self.monitor_op == np.less else -np.Inf
 
     def on_epoch_end(self, epoch, logs={}):
         current = logs.get(self.monitor)
@@ -369,6 +366,7 @@ class RemoteMonitor(Callback):
             of event data.
     '''
     def __init__(self, root='http://localhost:9000'):
+        super(RemoteMonitor, self).__init__()
         self.root = root
 
     def on_epoch_end(self, epoch, logs={}):
@@ -426,19 +424,23 @@ class TensorBoard(Callback):
 
     # Arguments
         log_dir: the path of the directory where to save the log
-            files to be parsed by tensorboard
+            files to be parsed by Tensorboard
         histogram_freq: frequency (in epochs) at which to compute activation
             histograms for the layers of the model. If set to 0,
             histograms won't be computed.
+        write_graph: whether to visualize the graph in Tensorboard. The log file can
+            become quite large when write_graph is set to True.
     '''
-    def __init__(self, log_dir='./logs', histogram_freq=0):
-        super(Callback, self).__init__()
+
+    def __init__(self, log_dir='./logs', histogram_freq=0, write_graph=True):
+        super(TensorBoard, self).__init__()
         if K._BACKEND != 'tensorflow':
             raise Exception('TensorBoard callback only works '
                             'with the TensorFlow backend.')
         self.log_dir = log_dir
         self.histogram_freq = histogram_freq
         self.merged = None
+        self.write_graph = write_graph
 
     def _set_model(self, model):
         import tensorflow as tf
@@ -457,8 +459,15 @@ class TensorBoard(Callback):
                     tf.histogram_summary('{}_out'.format(layer),
                                          layer.output)
         self.merged = tf.merge_all_summaries()
-        self.writer = tf.train.SummaryWriter(self.log_dir,
-                                             self.sess.graph_def)
+        if self.write_graph:
+            if tf.__version__ >= '0.8.0':
+                self.writer = tf.train.SummaryWriter(self.log_dir,
+                                                     self.sess.graph)
+            else:
+                self.writer = tf.train.SummaryWriter(self.log_dir,
+                                                     self.sess.graph_def)
+        else:
+            self.writer = tf.train.SummaryWriter(self.log_dir)
 
     def on_epoch_end(self, epoch, logs={}):
         import tensorflow as tf

keras/datasets/cifar.py

@@ -2,7 +2,6 @@
 from __future__ import absolute_import
 import sys
 from six.moves import cPickle
-from six.moves import range
 
 
 def load_batch(fpath, label_key='labels'):

keras/datasets/reuters.py

@@ -4,6 +4,7 @@ from ..utils.data_utils import get_file
 from six.moves import cPickle
 from six.moves import zip
 import numpy as np
+import sys
 
 
 def load_data(path="reuters.pkl", nb_words=None, skip_top=0,
@@ -64,4 +65,11 @@ def load_data(path="reuters.pkl", nb_words=None, skip_top=0,
 def get_word_index(path="reuters_word_index.pkl"):
     path = get_file(path, origin="https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl")
     f = open(path, 'rb')
-    return cPickle.load(f)
+
+    if sys.version_info < (3,):
+        data = cPickle.load(f)
+    else:
+        data = cPickle.load(f, encoding="latin1")
+
+    f.close()
+    return data

keras/engine/topology.py

@@ -281,7 +281,7 @@ class Layer(object):
         self.outbound_nodes = []
 
         # these properties will be set upon call of self.build(),
-        # which itself will be calld upon self.add_inbound_node if necessary.
+        # which itself will be called upon self.add_inbound_node if necessary.
         self.trainable_weights = []
         self.non_trainable_weights = []
         self.regularizers = []
@@ -512,7 +512,7 @@ class Layer(object):
                 where to connect the current layer.
             tensor_indices: integer or list of integers.
                 The output of the inbound node might be a list/tuple
-                of tensor, and we might only be interested in one sepcific entry.
+                of tensor, and we might only be interested in one specific entry.
                 This index allows you to specify the index of the entry in the output list
                 (if applicable). "None" means that we take all outputs (as a list).
         '''
@@ -847,10 +847,11 @@ class Layer(object):
         if not params:
             return
         weight_value_tuples = []
-        for p, w in zip(params, weights):
-            if K.get_value(p).shape != w.shape:
+        param_values = K.batch_get_value(params)
+        for pv, p, w in zip(param_values, params, weights):
+            if pv.shape != w.shape:
                 raise Exception('Layer weight shape ' +
-                                str(K.get_value(p).shape) +
+                                str(pv.shape) +
                                 ' not compatible with '
                                 'provided weight shape ' + str(w.shape))
             weight_value_tuples.append((p, w))
@@ -861,10 +862,7 @@ class Layer(object):
         as a list of numpy arrays.
         '''
         params = self.trainable_weights + self.non_trainable_weights
-        weights = []
-        for p in params:
-            weights.append(K.get_value(p))
-        return weights
+        return K.batch_get_value(params)
 
     def get_config(self):
         '''Returns a Python dictionary (serializable)
@@ -1005,7 +1003,7 @@ def Input(shape=None, batch_shape=None,
             Should be unique in a model (do not reuse the same name twice).
             It will be autogenerated if it isn't provided.
         dtype: The data type expected by the input, as a string
-            (`float32`, `flaot64`, `int32`...)
+            (`float32`, `float64`, `int32`...)
 
     # Example usage
 
@@ -1017,9 +1015,9 @@ def Input(shape=None, batch_shape=None,
         ```
     '''
     if not batch_shape:
-        assert shape, ('Please provide to Input either an `input_shape`' +
-                       ' or `batch_input_shape` argument. Note that ' +
-                       '`input_shape` does not include the batch '
+        assert shape, ('Please provide to Input either a `shape`' +
+                       ' or a `batch_shape` argument. Note that ' +
+                       '`shape` does not include the batch '
                        'dimension.')
         batch_shape = (None,) + tuple(shape)
     input_layer = InputLayer(batch_input_shape=batch_shape,
@@ -1062,10 +1060,14 @@ class Merge(Layer):
             and return a single tensor.
         concat_axis: integer, axis to use in mode `concat`.
         dot_axes: integer or tuple of integers, axes to use in mode `dot`.
-        output_shape: shape tuple (tuple of integers), or lambda/function
-            to compute output_shape (only if merge mode is a lambda/function).
-            If the latter case, it should take as input a list of shape tuples
-            (1:1 mapping to input tensors) and return a single shape tuple.
+        output_shape: either a shape tuple (tuple of integers), or a lambda/function
+            to compute `output_shape` (only if merge mode is a lambda/function).
+            If the argument is a tuple,
+            it should be expected output shape, *not* including the batch size
+            (same convention as the `input_shape` argument in layers).
+            If the argument is callable, it should take as input a list of shape tuples
+            (1:1 mapping to input tensors) and return a single shape tuple, including the
+            batch size (same convention as the `get_output_shape_for` method of layers).
         node_indices: optional list of integers containing
             the output node index for each input layer
             (in case some input layers have multiple output nodes).
@@ -1073,9 +1075,12 @@ class Merge(Layer):
         tensor_indices: optional list of indices of output tensors
             to consider for merging
             (in case some input layer node returns multiple tensors).
+        output_mask: mask or lambda/function to compute the output mask (only
+            if merge mode is a lambda/function). If the latter case, it should
+            take as input a list of masks and return a single mask.
     '''
     def __init__(self, layers=None, mode='sum', concat_axis=-1,
-                 dot_axes=-1, output_shape=None,
+                 dot_axes=-1, output_shape=None, output_mask=None,
                  node_indices=None, tensor_indices=None, name=None):
         self.layers = layers
         self.mode = mode
@@ -1085,6 +1090,7 @@ class Merge(Layer):
             self.dot_axes = [self.dot_axes, ] * 2
         self._output_shape = output_shape
         self.node_indices = node_indices
+        self._output_mask = output_mask
 
         # layer parameters
         self.inbound_nodes = []
@@ -1093,7 +1099,7 @@ class Merge(Layer):
         self.regularizers = []
         self.trainable_weights = []
         self.non_trainable_weights = []
-        self.supports_masking = False
+        self.supports_masking = True
         self.uses_learning_phase = False
         self.input_spec = None  # compatible with whatever
         if not name:
@@ -1112,7 +1118,6 @@ class Merge(Layer):
                 node_indices = [0 for _ in range(len(layers))]
             self._arguments_validation(layers, mode,
                                        concat_axis, dot_axes,
-                                       output_shape,
                                        node_indices, tensor_indices)
             self.built = True
             self.add_inbound_node(layers, node_indices, tensor_indices)
@@ -1120,7 +1125,7 @@ class Merge(Layer):
             self.built = False
 
     def _arguments_validation(self, layers, mode, concat_axis, dot_axes,
-                              output_shape, node_indices, tensor_indices):
+                              node_indices, tensor_indices):
         '''Validates user-passed arguments and raises exceptions
         as appropriate.
         '''
@@ -1128,7 +1133,8 @@ class Merge(Layer):
             if mode not in {'sum', 'mul', 'concat', 'ave', 'cos', 'dot'}:
                 raise Exception('Invalid merge mode: ' + str(mode))
         if type(layers) not in {list, tuple} or len(layers) < 2:
-            raise Exception('A Merge should only be applied to a list of layers. Not a list: ' + str(layers))
+            raise Exception('A Merge should only be applied to a list of '
+                            'layers with at least 2 elements. Found: ' + str(layers))
 
         if tensor_indices is None:
             tensor_indices = [None for _ in range(len(layers))]
@@ -1169,7 +1175,7 @@ class Merge(Layer):
                     raise Exception('Invalid format for dot_axes - list elements should be "int".')
                 if shape1[dot_axes[0]] != shape2[dot_axes[1]]:
                     raise Exception('Dimension incompatibility using dot mode: ' +
-                                    '%s != %s. ' % (shape1[dot_axes[0]], shape2[dot_axes[1][i]]) +
+                                    '%s != %s. ' % (shape1[dot_axes[0]], shape2[dot_axes[1]]) +
                                     'Layer shapes: %s, %s' % (shape1, shape2))
         elif mode == 'concat':
             reduced_inputs_shapes = [list(shape) for shape in input_shapes]
@@ -1221,6 +1227,7 @@ class Merge(Layer):
             l2 = inputs[1]
             denominator = K.sqrt(K.batch_dot(l1, l1, self.dot_axes) *
                                  K.batch_dot(l2, l2, self.dot_axes))
+            denominator = K.maximum(denominator, K.epsilon())
             output = K.batch_dot(l1, l2, self.dot_axes) / denominator
             output = K.expand_dims(output, 1)
             return output
@@ -1230,8 +1237,8 @@ class Merge(Layer):
     def __call__(self, inputs, mask=None):
         '''We disable successive calls to __call__ for Merge layers.
         Although there is no technical obstacle to
-        making it possible to __call__ a Merge intance many times
-        (it is just a layer), it would make for a rather unelegant API.
+        making it possible to __call__ a Merge instance many times
+        (it is just a layer), it would make for a rather inelegant API.
         '''
         if type(inputs) is not list:
             raise Exception('Merge can only be called on a list of tensors, '
@@ -1241,30 +1248,42 @@ class Merge(Layer):
                             'please use ' +
                             'the "merge" function instead: ' +
                             '`merged_tensor = merge([tensor_1, tensor2])`.')
-        layers = []
-        node_indices = []
-        tensor_indices = []
+
+        all_keras_tensors = True
         for x in inputs:
-            layer, node_index, tensor_index = x._keras_history
-            layers.append(layer)
-            node_indices.append(node_index)
-            tensor_indices.append(tensor_index)
-        self._arguments_validation(layers, self.mode,
-                                   self.concat_axis, self.dot_axes,
-                                   self._output_shape,
-                                   node_indices, tensor_indices)
-        self.built = True
-        self.add_inbound_node(layers, node_indices, tensor_indices)
+            if not hasattr(x, '_keras_history'):
+                all_keras_tensors = False
+                break
+
+        if all_keras_tensors:
+            layers = []
+            node_indices = []
+            tensor_indices = []
+            for x in inputs:
+                layer, node_index, tensor_index = x._keras_history
+                layers.append(layer)
+                node_indices.append(node_index)
+                tensor_indices.append(tensor_index)
+            self._arguments_validation(layers, self.mode,
+                                       self.concat_axis, self.dot_axes,
+                                       node_indices, tensor_indices)
+            self.built = True
+            self.add_inbound_node(layers, node_indices, tensor_indices)
+
+            outputs = self.inbound_nodes[-1].output_tensors
+            return outputs[0]  # merge only returns a single tensor
+        else:
+            return self.call(inputs, mask)
 
     def get_output_shape_for(self, input_shape):
-        assert type(input_shape) is list  # must have mutiple input shape tuples
+        assert type(input_shape) is list  # must have multiple input shape tuples
         # case: callable self._output_shape
         if hasattr(self.mode, '__call__'):
             if hasattr(self._output_shape, '__call__'):
                 output_shape = self._output_shape(input_shape)
                 return output_shape
             elif self._output_shape is not None:
-                return self._output_shape
+                return (input_shape[0],) + tuple(self._output_shape)
             else:
                 # TODO: consider shape auto-inference with TF
                 raise Exception('The Merge layer ' + self.name +
@@ -1286,12 +1305,10 @@ class Merge(Layer):
                     break
                 output_shape[self.concat_axis] += shape[self.concat_axis]
             return tuple(output_shape)
-        elif self.mode == 'join':
-            return None
         elif self.mode == 'dot':
             shape1 = list(input_shapes[0])
             shape2 = list(input_shapes[1])
-            dot_axes = [a-1 for a in self.dot_axes]
+            dot_axes = [a - 1 for a in self.dot_axes]
             tensordot_output = np.tensordot(np.zeros(tuple(shape1[1:])),
                                             np.zeros(tuple(shape2[1:])),
                                             axes=dot_axes)
@@ -1303,22 +1320,39 @@ class Merge(Layer):
         elif self.mode == 'cos':
             return (input_shapes[0][0], 1)
 
-    def compute_mask(self, input, mask=None):
-        '''TODO: add mask merging support
-        '''
-        if mask is not None and any(mask):
-            raise Exception('Merge does not support masking, ' +
-                            'but was passed an input mask: ' + str(mask))
-        return None
+    def compute_mask(self, inputs, mask=None):
+        if mask is None or not any([m is not None for m in mask]):
+            return None
+
+        assert hasattr(mask, '__len__') and len(mask) == len(inputs)
+
+        if self.mode in ['sum', 'mul', 'ave']:
+            masks = [K.expand_dims(m, 0) for m in mask if m is not None]
+            return K.all(K.concatenate(masks, axis=0), axis=0, keepdims=False)
+        elif self.mode == 'concat':
+            masks = [K.ones_like(inputs[i][:-1]) if m is None else m for i, m in zip(inputs, mask)]
+            expanded_dims = [K.expand_dims(m) for m in masks]
+            concatenated = K.concatenate(expanded_dims, axis=self.concat_axis)
+            return K.all(concatenated, axis=-1, keepdims=False)
+        elif self.mode in ['cos', 'dot']:
+            return None
+        elif hasattr(self.mode, '__call__'):
+            if hasattr(self._output_mask, '__call__'):
+                return self._output_mask(mask)
+            else:
+                return self._output_mask
+        else:
+            # this should have been caught earlier
+            raise Exception('Invalid merge mode: {}'.format(self.mode))
 
     def get_config(self):
         py3 = sys.version_info[0] == 3
 
         if isinstance(self.mode, python_types.LambdaType):
             if py3:
-                mode = marshal.dumps(self.mode.__code__)
+                mode = marshal.dumps(self.mode.__code__).decode('raw_unicode_escape')
             else:
-                mode = marshal.dumps(self.mode.func_code)
+                mode = marshal.dumps(self.mode.func_code).decode('raw_unicode_escape')
             mode_type = 'lambda'
         elif callable(self.mode):
             mode = self.mode.__name__
@@ -1329,9 +1363,9 @@ class Merge(Layer):
 
         if isinstance(self._output_shape, python_types.LambdaType):
             if py3:
-                output_shape = marshal.dumps(self._output_shape.__code__)
+                output_shape = marshal.dumps(self._output_shape.__code__).decode('raw_unicode_escape')
             else:
-                output_shape = marshal.dumps(self._output_shape.func_code)
+                output_shape = marshal.dumps(self._output_shape.func_code).decode('raw_unicode_escape')
             output_shape_type = 'lambda'
         elif callable(self._output_shape):
             output_shape = self._output_shape.__name__
@@ -1354,7 +1388,7 @@ class Merge(Layer):
         if mode_type == 'function':
             mode = globals()[config['mode']]
         elif mode_type == 'lambda':
-            mode = marshal.loads(config['mode'])
+            mode = marshal.loads(config['mode'].encode('raw_unicode_escape'))
             mode = python_types.FunctionType(mode, globals())
         else:
             mode = config['mode']
@@ -1363,7 +1397,7 @@ class Merge(Layer):
         if output_shape_type == 'function':
             output_shape = globals()[config['output_shape']]
         elif output_shape_type == 'lambda':
-            output_shape = marshal.loads(config['output_shape'])
+            output_shape = marshal.loads(config['output_shape'].encode('raw_unicode_escape'))
             output_shape = python_types.FunctionType(output_shape, globals())
         else:
             output_shape = config['output_shape']
@@ -1374,7 +1408,7 @@ class Merge(Layer):
 
 
 def merge(inputs, mode='sum', concat_axis=-1,
-          dot_axes=-1, output_shape=None, name=None):
+          dot_axes=-1, output_shape=None, output_mask=None, name=None):
     '''Functional merge, to apply to Keras tensors (NOT layers).
     Returns a Keras tensor.
 
@@ -1388,7 +1422,7 @@ def merge(inputs, mode='sum', concat_axis=-1,
 
     # Arguments
         mode: string or lambda/function. If string, must be one
-            of: 'sum', 'mul', 'concat', 'ave', 'join', 'cos', 'dot'.
+            of: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot'.
             If lambda/function, it should take as input a list of tensors
             and return a single tensor.
         concat_axis: integer, axis to use in mode `concat`.
@@ -1396,7 +1430,8 @@ def merge(inputs, mode='sum', concat_axis=-1,
         output_shape: shape tuple (tuple of integers), or lambda/function
             to compute output_shape (only if merge mode is a lambda/function).
             If the latter case, it should take as input a list of shape tuples
-            (1:1 mapping to input tensors) and return a single shape tuple.
+            (1:1 mapping to input tensors) and return a single shape tuple, including the
+            batch size (same convention as the `get_output_shape_for` method of layers).
         node_indices: optional list of integers containing
             the output node index for each input layer
             (in case some input layers have multiple output nodes).
@@ -1405,20 +1440,37 @@ def merge(inputs, mode='sum', concat_axis=-1,
             to consider for merging
             (in case some input layer node returns multiple tensors).
     '''
-    input_layers = []
-    node_indices = []
-    tensor_indices = []
+    all_keras_tensors = True
     for x in inputs:
-        assert hasattr(x, '_keras_history'), 'Input tensor to "merge" was not a Keras tensor: ' + str(x)
-        input_layer, node_index, tensor_index = x._keras_history
-        input_layers.append(input_layer)
-        node_indices.append(node_index)
-        tensor_indices.append(tensor_index)
-    merge_layer = Merge(input_layers, mode=mode, concat_axis=concat_axis,
-                        dot_axes=dot_axes, output_shape=output_shape,
-                        node_indices=node_indices, tensor_indices=tensor_indices,
-                        name=name)
-    return merge_layer.inbound_nodes[0].output_tensors[0]
+        if not hasattr(x, '_keras_history'):
+            all_keras_tensors = False
+            break
+    if all_keras_tensors:
+        input_layers = []
+        node_indices = []
+        tensor_indices = []
+        for x in inputs:
+            input_layer, node_index, tensor_index = x._keras_history
+            input_layers.append(input_layer)
+            node_indices.append(node_index)
+            tensor_indices.append(tensor_index)
+        merge_layer = Merge(input_layers, mode=mode,
+                            concat_axis=concat_axis,
+                            dot_axes=dot_axes,
+                            output_shape=output_shape,
+                            output_mask=output_mask,
+                            node_indices=node_indices,
+                            tensor_indices=tensor_indices,
+                            name=name)
+        return merge_layer.inbound_nodes[0].output_tensors[0]
+    else:
+        merge_layer = Merge(mode=mode,
+                            concat_axis=concat_axis,
+                            dot_axes=dot_axes,
+                            output_shape=output_shape,
+                            output_mask=output_mask,
+                            name=name)
+        return merge_layer(inputs)
 
 
 class Container(Layer):
@@ -1541,12 +1593,28 @@ class Container(Layer):
                 raise Exception('Output tensors to a ' + cls_name + ' must be '
                                 'Keras tensors. Found: ' + str(x))
         # build self.output_layers:
+        masks = []
         for x in self.outputs:
             layer, node_index, tensor_index = x._keras_history
             self.output_layers.append(layer)
             self.output_layers_node_indices.append(node_index)
             self.output_layers_tensor_indices.append(tensor_index)
-        # build self.output_layers:
+
+            # also fill in the output mask cache
+            node = layer.inbound_nodes[node_index]
+            mask = node.output_masks[tensor_index]
+            masks.append(mask)
+
+        # output mask cache
+        mask_cache_key = ','.join([str(id(x)) for x in self.inputs])
+        mask_cache_key += '_' + ','.join([str(id(x)) for x in masks])
+        if len(masks) == 1:
+            mask = masks[0]
+        else:
+            mask = masks
+        self._output_mask_cache[mask_cache_key] = mask
+
+        # build self.input_layers:
         for x in self.inputs:
             layer, node_index, tensor_index = x._keras_history
             # it's supposed to be an input layer, so only one node
@@ -1574,7 +1642,7 @@ class Container(Layer):
         nodes_depths = {}  # map {node: depth value}
         layers_depths = {}  # map {layer: depth value}
 
-        def make_node_key(node, depth):
+        def make_node_marker(node, depth):
             return str(id(node)) + '-' + str(depth)
 
         def build_map_of_graph(tensor, seen_nodes=set(), depth=0,
@@ -1598,7 +1666,7 @@ class Container(Layer):
             node = layer.inbound_nodes[node_index]
 
             # prevent cycles
-            seen_nodes.add(make_node_key(node, depth))
+            seen_nodes.add(make_node_marker(node, depth))
 
             node_key = layer.name + '_ib-' + str(node_index)
             # update container_nodes
@@ -1610,11 +1678,12 @@ class Container(Layer):
             else:
                 nodes_depths[node] = max(depth, node_depth)
             # update layers_depths
-            layer_depth = layers_depths.get(layer)
-            if layer_depth is None:
-                layers_depths[layer] = depth
+            previously_seen_depth = layers_depths.get(layer)
+            if previously_seen_depth is None:
+                current_depth = depth
             else:
-                layers_depths[layer] = max(depth, layer_depth)
+                current_depth = max(depth, previously_seen_depth)
+            layers_depths[layer] = current_depth
 
             # propagate to all previous tensors connected to this node
             for i in range(len(node.inbound_layers)):
@@ -1623,9 +1692,10 @@ class Container(Layer):
                 node_index = node.node_indices[i]
                 tensor_index = node.tensor_indices[i]
                 next_node = layer.inbound_nodes[node_index]
-                node_key = make_node_key(next_node, depth + 1)
-                if node_key not in seen_nodes:
-                    build_map_of_graph(x, seen_nodes, depth + 1,
+                # use node_marker to prevent cycles
+                node_marker = make_node_marker(next_node, current_depth + 1)
+                if node_marker not in seen_nodes:
+                    build_map_of_graph(x, seen_nodes, current_depth + 1,
                                        layer, node_index, tensor_index)
 
         for x in self.outputs:
@@ -1646,7 +1716,8 @@ class Container(Layer):
                 layers_by_depth[depth] = []
             layers_by_depth[depth].append(layer)
 
-        depth_keys = list(nodes_by_depth.keys())
+        # get sorted list of layer depths
+        depth_keys = list(layers_by_depth.keys())
         depth_keys.sort(reverse=True)
 
         # set self.layers and self.layers_by_depth
@@ -1660,6 +1731,10 @@ class Container(Layer):
         self.layers = layers
         self.layers_by_depth = layers_by_depth
 
+        # get sorted list of node depths
+        depth_keys = list(nodes_by_depth.keys())
+        depth_keys.sort(reverse=True)
+
         # check that all tensors required are computable.
         # computable_tensors: all tensors in the graph
         # that can be computed from the inputs provided
@@ -2075,8 +2150,6 @@ class Container(Layer):
         return output_tensors, output_masks, output_shapes
 
     def get_config(self):
-        '''TODO: add keras version information
-        '''
         config = {
             'name': self.name,
         }
@@ -2157,9 +2230,9 @@ class Container(Layer):
         # the graph reconstruction process
         created_layers = {}
 
-        # iterate over saved layers, instantiate them,
-        # then call them on appropriate inputs to create graph nodes
-        for layer_data in config['layers']:
+        def process_layer(layer_data):
+            # iterate over saved layers, instantiate them,
+            # then call them on appropriate inputs to create graph nodes
             layer_name = layer_data['name']
 
             # instantiate layer
@@ -2184,6 +2257,10 @@ class Container(Layer):
                         layer(input_tensors[0])
                     else:
                         layer(input_tensors)
+
+        for layer_data in config['layers']:
+            process_layer(layer_data)
+
         name = config.get('name')
         input_tensors = []
         output_tensors = []
@@ -2241,7 +2318,7 @@ class Container(Layer):
         for layer in flattened_layers:
             g = f.create_group(layer.name)
             symbolic_weights = layer.trainable_weights + layer.non_trainable_weights
-            weight_values = layer.get_weights()
+            weight_values = K.batch_get_value(symbolic_weights)
             weight_names = []
             for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
                 if hasattr(w, 'name') and w.name:
@@ -2316,6 +2393,26 @@ class Container(Layer):
             K.batch_set_value(weight_value_tuples)
         f.close()
 
+    def _updated_config(self):
+        '''shared between different serialization methods'''
+        from keras import __version__ as keras_version
+
+        config = self.get_config()
+        model_config = {
+            'class_name': self.__class__.__name__,
+            'config': config,
+            'keras_version': keras_version
+        }
+
+        if hasattr(self, 'optimizer'):
+            model_config['optimizer'] = self.optimizer.get_config()
+            model_config['loss'] = getattr(self.loss, '__name__', self.loss)
+            model_config['sample_weight_mode'] = self.sample_weight_mode
+
+        if hasattr(self, 'loss_weights'):
+            model_config['loss_weights'] = self.loss_weights
+        return model_config
+
     def to_json(self, **kwargs):
         '''Returns a JSON string containing the network configuration.
 
@@ -2335,11 +2432,7 @@ class Container(Layer):
 
             raise TypeError('Not JSON Serializable')
 
-        config = self.get_config()
-        model_config = {
-            'class_name': self.__class__.__name__,
-            'config': config,
-        }
+        model_config = self._updated_config()
         return json.dumps(model_config, default=get_json_type, **kwargs)
 
     def to_yaml(self, **kwargs):
@@ -2353,14 +2446,9 @@ class Container(Layer):
         functions / classes.
         '''
         import yaml
-        config = self.get_config()
-        model_config = {
-            'class_name': self.__class__.__name__,
-            'config': config,
-        }
-        return yaml.dump(model_config, **kwargs)
+        return yaml.dump(self._updated_config(), **kwargs)
 
-    def summary(self):
+    def summary(self, line_length=100, positions=[.33, .55, .67, 1.]):
         from keras.utils.layer_utils import print_summary
 
         if hasattr(self, 'flattened_layers'):
@@ -2369,7 +2457,7 @@ class Container(Layer):
         else:
             flattened_layers = self.layers
 
-        print_summary(flattened_layers, getattr(self, 'container_nodes', None))
+        print_summary(flattened_layers, getattr(self, 'container_nodes', None), line_length=line_length, positions=positions)
 
 
 def get_source_inputs(tensor, layer=None, node_index=None):

keras/engine/training.py

@@ -20,7 +20,8 @@ from ..utils.generic_utils import Progbar
 from .. import callbacks as cbks
 
 
-def standardize_input_data(data, names, shapes=None, check_batch_dim=True,
+def standardize_input_data(data, names, shapes=None,
+                           check_batch_dim=True,
                            exception_prefix=''):
     '''Users may pass data as a list of arrays, dictionary of arrays,
     or as a single array. We normalize this to an ordered list of
@@ -54,7 +55,7 @@ def standardize_input_data(data, names, shapes=None, check_batch_dim=True,
                     raise Exception('Error when checking ' + exception_prefix +
                                     ': you are passing a list as '
                                     'input to your model, '
-                                    'but the model expects a '
+                                    'but the model expects '
                                     'a list of ' + str(len(names)) +
                                     ' Numpy arrays instead. '
                                     'The list you passed was: ' +
@@ -66,6 +67,12 @@ def standardize_input_data(data, names, shapes=None, check_batch_dim=True,
                             ': data should be a Numpy array, '
                             'or list/dict of Numpy arrays. '
                             'Found: ' + str(data)[:200] + '...')
+        if len(names) != 1:
+            # case: model expects multiple inputs but only received
+            # a single Numpy array
+            raise Exception('The model expects ' + str(len(names)) +
+                            ' input arrays, but only received one array. '
+                            'Found: array with shape ' + str(data.shape))
         arrays = [data]
 
     # make arrays at least 2D
@@ -78,8 +85,7 @@ def standardize_input_data(data, names, shapes=None, check_batch_dim=True,
     # check shapes compatibility
     if shapes:
         for i in range(len(names)):
-            if not i and not check_batch_dim:
-                # skip the first axis
+            if shapes[i] is None:
                 continue
             array = arrays[i]
             if len(array.shape) != len(shapes[i]):
@@ -88,7 +94,10 @@ def standardize_input_data(data, names, shapes=None, check_batch_dim=True,
                                 ' to have ' + str(len(shapes[i])) +
                                 ' dimensions, but got array with shape ' +
                                 str(array.shape))
-            for dim, ref_dim in zip(array.shape, shapes[i]):
+            for j, (dim, ref_dim) in enumerate(zip(array.shape, shapes[i])):
+                if not j and not check_batch_dim:
+                    # skip the first axis
+                    continue
                 if ref_dim:
                     if ref_dim != dim:
                         raise Exception('Error when checking ' + exception_prefix +
@@ -153,7 +162,7 @@ def check_array_lengths(X, Y, W):
         raise Exception('All input arrays (x) should have '
                         'the same number of samples.')
     set_y = set(y_lengths)
-    if len(set_x) != 1:
+    if len(set_y) != 1:
         raise Exception('All target arrays (y) should have '
                         'the same number of samples.')
     set_w = set(w_lengths)
@@ -195,7 +204,7 @@ def check_loss_and_target_compatibility(targets, losses, output_shapes):
                                 'Alternatively, you can use the loss function '
                                 '`sparse_categorical_crossentropy` instead, '
                                 'which does expect integer targets.')
-        if loss.__name__ in key_losses and y.shape[1] != shape[1]:
+        if loss.__name__ in key_losses and shape[1] is not None and y.shape[1] != shape[1]:
                 raise Exception('A target array with shape ' + str(y.shape) +
                                 ' was passed for an output of shape ' + str(shape) +
                                 ' while using as loss `' + loss.__name__ + '`. '
@@ -225,11 +234,17 @@ def collect_metrics(metrics, output_names):
 
 
 def collect_trainable_weights(layer):
+    '''Collects all `trainable_weights` attributes,
+    excluding any sublayers where `trainable` is set the `False`.
+    '''
     trainable = getattr(layer, 'trainable', True)
     if not trainable:
         return []
     weights = []
-    if layer.__class__.__name__ in ['Sequential', 'Model']:
+    if layer.__class__.__name__ == 'Sequential':
+        for sublayer in layer.flattened_layers:
+            weights += collect_trainable_weights(sublayer)
+    elif layer.__class__.__name__ == 'Model':
         for sublayer in layer.layers:
             weights += collect_trainable_weights(sublayer)
     elif layer.__class__.__name__ == 'Graph':
@@ -237,6 +252,9 @@ def collect_trainable_weights(layer):
             weights += collect_trainable_weights(sublayer)
     else:
         weights += layer.trainable_weights
+    # dedupe weights
+    weights = list(set(weights))
+    weights.sort(key=lambda x: x.name)
     return weights
 
 
@@ -446,6 +464,7 @@ class Model(Container):
         self.optimizer = optimizers.get(optimizer)
         self.sample_weight_mode = sample_weight_mode
         self.loss = loss
+        self.loss_weights = loss_weights
 
         # prepare loss weights
         if loss_weights is None:
@@ -565,6 +584,10 @@ class Model(Container):
             name = self.output_names[i]
             self.targets.append(K.placeholder(ndim=len(shape), name=name + '_target'))
 
+        # prepare metrics
+        self.metrics_names = ['loss']
+        self.metrics = []
+
         # compute total loss
         total_loss = None
         for i in range(len(self.outputs)):
@@ -574,19 +597,20 @@ class Model(Container):
             sample_weight = sample_weights[i]
             mask = masks[i]
             loss_weight = loss_weights_list[i]
-            output_loss = loss_weight * weighted_loss(y_true, y_pred,
-                                                      sample_weight, mask)
+            output_loss = weighted_loss(y_true, y_pred,
+                                        sample_weight, mask)
+            if len(self.outputs) > 1:
+                self.metrics.append(output_loss)
+                self.metrics_names.append(self.output_names[i] + '_loss')
             if total_loss is None:
-                total_loss = output_loss
+                total_loss = loss_weight * output_loss
             else:
-                total_loss += output_loss
+                total_loss += loss_weight * output_loss
+
         # add regularization penalties to the loss
         for r in self.regularizers:
             total_loss = r(total_loss)
 
-        # prepare metrics
-        self.metrics_names = ['loss']
-        self.metrics = []
         # list of same size as output_names.
         # contains tuples (metrics for output, names of metrics)
         nested_metrics = collect_metrics(metrics, self.output_names)
@@ -602,8 +626,12 @@ class Model(Container):
                     if output_shape[-1] == 1:
                         # case: binary accuracy
                         self.metrics.append(metrics_module.binary_accuracy(y_true, y_pred))
+                    elif self.loss_functions[i] == objectives.sparse_categorical_crossentropy:
+                        # case: categorical accuracy with sparse targets
+                        self.metrics.append(
+                            metrics_module.sparse_categorical_accuracy(y_true, y_pred))
                     else:
-                        # case: categorical accuracy
+                        # case: categorical accuracy with dense targets
                         self.metrics.append(metrics_module.categorical_accuracy(y_true, y_pred))
                     if len(self.output_names) == 1:
                         self.metrics_names.append('acc')
@@ -632,6 +660,8 @@ class Model(Container):
         self.predict_function = None
 
     def _make_train_function(self):
+        if not hasattr(self, 'train_function'):
+            raise Exception('You must compile your model before using it.')
         if self.train_function is None:
             if self.uses_learning_phase:
                 inputs = self.inputs + self.targets + self.sample_weights + [K.learning_phase()]
@@ -639,10 +669,7 @@ class Model(Container):
                 inputs = self.inputs + self.targets + self.sample_weights
 
             # get trainable weights
-            trainable_weights = []
-            for layer in self.layers:
-                trainable_weights += collect_trainable_weights(layer)
-
+            trainable_weights = collect_trainable_weights(self)
             training_updates = self.optimizer.get_updates(trainable_weights, self.constraints, self.total_loss)
             updates = self.updates + training_updates
 
@@ -653,6 +680,8 @@ class Model(Container):
                                              **self._function_kwargs)
 
     def _make_test_function(self):
+        if not hasattr(self, 'test_function'):
+            raise Exception('You must compile your model before using it.')
         if self.test_function is None:
             if self.uses_learning_phase:
                 inputs = self.inputs + self.targets + self.sample_weights + [K.learning_phase()]
@@ -666,6 +695,8 @@ class Model(Container):
                                             **self._function_kwargs)
 
     def _make_predict_function(self):
+        if not hasattr(self, 'predict_function'):
+            self.predict_function = None
         if self.predict_function is None:
             if self.uses_learning_phase:
                 inputs = self.inputs + [K.learning_phase()]
@@ -673,10 +704,11 @@ class Model(Container):
                 inputs = self.inputs
             # returns network outputs. Does not update weights.
             # Does update the network states.
+            kwargs = getattr(self, '_function_kwargs', {})
             self.predict_function = K.function(inputs,
                                                self.outputs,
                                                updates=self.state_updates,
-                                               **self._function_kwargs)
+                                               **kwargs)
 
     def _fit_loop(self, f, ins, out_labels=[], batch_size=32,
                   nb_epoch=100, verbose=1, callbacks=[],
@@ -749,6 +781,7 @@ class Model(Container):
                 np.random.shuffle(index_array)
 
             batches = make_batches(nb_train_sample, batch_size)
+            epoch_logs = {}
             for batch_index, (batch_start, batch_end) in enumerate(batches):
                 batch_ids = index_array[batch_start:batch_end]
                 try:
@@ -773,7 +806,6 @@ class Model(Container):
 
                 callbacks.on_batch_end(batch_index, batch_logs)
 
-                epoch_logs = {}
                 if batch_index == len(batches) - 1:  # last batch
                     # validation
                     if do_validation:
@@ -802,7 +834,7 @@ class Model(Container):
             verbose: verbosity mode.
 
         # Returns
-            Array of prections (if the model has a single output)
+            Array of predictions (if the model has a single output)
             or list of arrays of predictions
             (if the model has multiple outputs).
         '''
@@ -892,12 +924,20 @@ class Model(Container):
             raise Exception('You must compile a model before training/testing.'
                             ' Use `model.compile(optimizer, loss)`.')
 
+        output_shapes = []
+        for output_shape, loss_fn in zip(self.internal_output_shapes, self.loss_functions):
+            if loss_fn.__name__ == 'sparse_categorical_crossentropy':
+                output_shapes.append(output_shape[:-1] + (1,))
+            elif getattr(objectives, loss_fn.__name__, None) is None:
+                output_shapes.append(None)
+            else:
+                output_shapes.append(output_shape)
         x = standardize_input_data(x, self.input_names,
                                    self.internal_input_shapes,
                                    check_batch_dim=False,
                                    exception_prefix='model input')
         y = standardize_input_data(y, self.output_names,
-                                   self.internal_output_shapes,
+                                   output_shapes,
                                    check_batch_dim=False,
                                    exception_prefix='model target')
         sample_weights = standardize_sample_weights(sample_weight,
@@ -946,7 +986,7 @@ class Model(Container):
                 at the end of each epoch. The model will not be trained on this data.
                 This could be a tuple (x_val, y_val) or a tuple (val_x, val_y, val_sample_weights).
             shuffle: boolean, whether to shuffle the training data before each epoch.
-            class_weight: optional dictionary mapping classe indices (integers) to
+            class_weight: optional dictionary mapping class indices (integers) to
                 a weight (float) to apply to the model's loss for the samples
                 from this class during training.
                 This can be useful to tell the model to "pay more attention" to
@@ -1017,6 +1057,18 @@ class Model(Container):
 
         # prepare display labels
         out_labels = self.metrics_names
+
+        # rename duplicated metrics name
+        # (can happen with an output layer shared among multiple dataflows)
+        deduped_out_labels = []
+        for i, label in enumerate(out_labels):
+            new_label = label
+            if out_labels.count(label) > 1:
+                dup_idx = out_labels[:i].count(label)
+                new_label += '_' + str(dup_idx + 1)
+            deduped_out_labels.append(new_label)
+        out_labels = deduped_out_labels
+
         if do_validation:
             callback_metrics = copy.copy(out_labels) + ['val_' + n for n in out_labels]
         else:
@@ -1031,7 +1083,7 @@ class Model(Container):
 
     def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):
         '''Returns the loss value and metrics values for the model
-        in test mode. Computation in done in batches.
+        in test mode. Computation is done in batches.
 
         # Arguments
             x: Numpy array of test data,
@@ -1121,7 +1173,7 @@ class Model(Container):
                 with shape (samples, sequence_length),
                 to apply a different weight to every timestep of every sample.
                 In this case you should make sure to specify sample_weight_mode="temporal" in compile().
-            class_weight: optional dictionary mapping classe indices (integers) to
+            class_weight: optional dictionary mapping class indices (integers) to
                 a weight (float) to apply to the model's loss for the samples
                 from this class during training.
                 This can be useful to tell the model to "pay more attention" to
@@ -1359,9 +1411,9 @@ class Model(Container):
                     outs = self.train_on_batch(x, y,
                                                sample_weight=sample_weight,
                                                class_weight=class_weight)
-                except Exception as e:
+                except:
                     _stop.set()
-                    raise e
+                    raise
 
                 if type(outs) != list:
                     outs = [outs]
@@ -1461,9 +1513,9 @@ class Model(Container):
                                 'or (x, y). Found: ' + str(generator_output))
             try:
                 outs = self.test_on_batch(x, y, sample_weight=sample_weight)
-            except Exception as e:
+            except:
                 _stop.set()
-                raise e
+                raise
 
             if type(x) is list:
                 nb_samples = len(x[0])
@@ -1533,9 +1585,9 @@ class Model(Container):
 
             try:
                 outs = self.predict_on_batch(x)
-            except Exception as e:
+            except:
                 _stop.set()
-                raise e
+                raise
 
             if type(x) is list:
                 nb_samples = len(x[0])

keras/initializations.py

@@ -12,11 +12,13 @@ def get_fans(shape, dim_ordering='th'):
         # TH kernel shape: (depth, input_depth, ...)
         # TF kernel shape: (..., input_depth, depth)
         if dim_ordering == 'th':
-            fan_in = np.prod(shape[1:])
-            fan_out = shape[0]
+            receptive_field_size = np.prod(shape[2:])
+            fan_in = shape[1] * receptive_field_size
+            fan_out = shape[0] * receptive_field_size
         elif dim_ordering == 'tf':
-            fan_in = np.prod(shape[:-1])
-            fan_out = shape[-1]
+            receptive_field_size = np.prod(shape[:2])
+            fan_in = shape[-2] * receptive_field_size
+            fan_out = shape[-1] * receptive_field_size
         else:
             raise Exception('Invalid dim_ordering: ' + dim_ordering)
     else:

keras/layers/advanced_activations.py

@@ -51,7 +51,7 @@ class PReLU(Layer):
 
     # Arguments
         init: initialization function for the weights.
-        weights: initial weights, as a list of a single numpy array.
+        weights: initial weights, as a list of a single Numpy array.
 
     # References
         - [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](http://arxiv.org/pdf/1502.01852v1.pdf)

keras/layers/convolutional.py

@@ -65,6 +65,7 @@ class Convolution1D(Layer):
             (eg. maxnorm, nonneg), applied to the main weights matrix.
         b_constraint: instance of the [constraints](../constraints.md) module,
             applied to the bias.
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
         input_dim: Number of channels/dimensions in the input.
             Either this argument or the keyword argument `input_shape`must be
             provided when using this layer as the first layer in a model.
@@ -85,7 +86,7 @@ class Convolution1D(Layer):
                  border_mode='valid', subsample_length=1,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
-                 input_dim=None, input_length=None, **kwargs):
+                 bias=True, input_dim=None, input_length=None, **kwargs):
 
         if border_mode not in {'valid', 'same'}:
             raise Exception('Invalid border mode for Convolution1D:', border_mode)
@@ -106,6 +107,7 @@ class Convolution1D(Layer):
         self.W_constraint = constraints.get(W_constraint)
         self.b_constraint = constraints.get(b_constraint)
 
+        self.bias = bias
         self.input_spec = [InputSpec(ndim=3)]
         self.initial_weights = weights
         self.input_dim = input_dim
@@ -118,15 +120,18 @@ class Convolution1D(Layer):
         input_dim = input_shape[2]
         self.W_shape = (self.nb_filter, input_dim, self.filter_length, 1)
         self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
-        self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
-        self.trainable_weights = [self.W, self.b]
+        if self.bias:
+            self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
+            self.trainable_weights = [self.W, self.b]
+        else:
+            self.trainable_weights = [self.W]
         self.regularizers = []
 
         if self.W_regularizer:
             self.W_regularizer.set_param(self.W)
             self.regularizers.append(self.W_regularizer)
 
-        if self.b_regularizer:
+        if self.bias and self.b_regularizer:
             self.b_regularizer.set_param(self.b)
             self.regularizers.append(self.b_regularizer)
 
@@ -137,7 +142,7 @@ class Convolution1D(Layer):
         self.constraints = {}
         if self.W_constraint:
             self.constraints[self.W] = self.W_constraint
-        if self.b_constraint:
+        if self.bias and self.b_constraint:
             self.constraints[self.b] = self.b_constraint
 
         if self.initial_weights is not None:
@@ -154,14 +159,14 @@ class Convolution1D(Layer):
     def call(self, x, mask=None):
         x = K.expand_dims(x, -1)  # add a dimension of the right
         x = K.permute_dimensions(x, (0, 2, 1, 3))
-        conv_out = K.conv2d(x, self.W, strides=self.subsample,
-                            border_mode=self.border_mode,
-                            dim_ordering='th')
-
-        output = conv_out + K.reshape(self.b, (1, self.nb_filter, 1, 1))
-        output = self.activation(output)
+        output = K.conv2d(x, self.W, strides=self.subsample,
+                          border_mode=self.border_mode,
+                          dim_ordering='th')
+        if self.bias:
+            output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
         output = K.squeeze(output, 3)  # remove the dummy 3rd dimension
         output = K.permute_dimensions(output, (0, 2, 1))
+        output = self.activation(output)
         return output
 
     def get_config(self):
@@ -176,6 +181,7 @@ class Convolution1D(Layer):
                   'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                   'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                   'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias,
                   'input_dim': self.input_dim,
                   'input_length': self.input_length}
         base_config = super(Convolution1D, self).get_config()
@@ -232,6 +238,10 @@ class Convolution2D(Layer):
             applied to the bias.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
 
     # Input shape
         4D tensor with shape:
@@ -248,9 +258,10 @@ class Convolution2D(Layer):
     '''
     def __init__(self, nb_filter, nb_row, nb_col,
                  init='glorot_uniform', activation='linear', weights=None,
-                 border_mode='valid', subsample=(1, 1), dim_ordering='th',
+                 border_mode='valid', subsample=(1, 1), dim_ordering=K.image_dim_ordering(),
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
-                 W_constraint=None, b_constraint=None, **kwargs):
+                 W_constraint=None, b_constraint=None,
+                 bias=True, **kwargs):
 
         if border_mode not in {'valid', 'same'}:
             raise Exception('Invalid border mode for Convolution2D:', border_mode)
@@ -272,6 +283,7 @@ class Convolution2D(Layer):
         self.W_constraint = constraints.get(W_constraint)
         self.b_constraint = constraints.get(b_constraint)
 
+        self.bias = bias
         self.input_spec = [InputSpec(ndim=4)]
         self.initial_weights = weights
         super(Convolution2D, self).__init__(**kwargs)
@@ -286,15 +298,18 @@ class Convolution2D(Layer):
         else:
             raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
         self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
-        self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
-        self.trainable_weights = [self.W, self.b]
+        if self.bias:
+            self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
+            self.trainable_weights = [self.W, self.b]
+        else:
+            self.trainable_weights = [self.W]
         self.regularizers = []
 
         if self.W_regularizer:
             self.W_regularizer.set_param(self.W)
             self.regularizers.append(self.W_regularizer)
 
-        if self.b_regularizer:
+        if self.bias and self.b_regularizer:
             self.b_regularizer.set_param(self.b)
             self.regularizers.append(self.b_regularizer)
 
@@ -305,7 +320,7 @@ class Convolution2D(Layer):
         self.constraints = {}
         if self.W_constraint:
             self.constraints[self.W] = self.W_constraint
-        if self.b_constraint:
+        if self.bias and self.b_constraint:
             self.constraints[self.b] = self.b_constraint
 
         if self.initial_weights is not None:
@@ -335,16 +350,17 @@ class Convolution2D(Layer):
             raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
 
     def call(self, x, mask=None):
-        conv_out = K.conv2d(x, self.W, strides=self.subsample,
-                            border_mode=self.border_mode,
-                            dim_ordering=self.dim_ordering,
-                            filter_shape=self.W_shape)
-        if self.dim_ordering == 'th':
-            output = conv_out + K.reshape(self.b, (1, self.nb_filter, 1, 1))
-        elif self.dim_ordering == 'tf':
-            output = conv_out + K.reshape(self.b, (1, 1, 1, self.nb_filter))
-        else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+        output = K.conv2d(x, self.W, strides=self.subsample,
+                          border_mode=self.border_mode,
+                          dim_ordering=self.dim_ordering,
+                          filter_shape=self.W_shape)
+        if self.bias:
+            if self.dim_ordering == 'th':
+                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
+            elif self.dim_ordering == 'tf':
+                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
         output = self.activation(output)
         return output
 
@@ -361,7 +377,8 @@ class Convolution2D(Layer):
                   'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
                   'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                   'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
-                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None}
+                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias}
         base_config = super(Convolution2D, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
@@ -377,7 +394,7 @@ class Convolution3D(Layer):
 
     # Arguments
         nb_filter: Number of convolution filters to use.
-        kernel_dim1: Length of the first dimension in the covolution kernel.
+        kernel_dim1: Length of the first dimension in the convolution kernel.
         kernel_dim2: Length of the second dimension in the convolution kernel.
         kernel_dim3: Length of the third dimension in the convolution kernel.
         init: name of initialization function for the weights of the layer
@@ -390,7 +407,7 @@ class Convolution3D(Layer):
             or alternatively, elementwise Theano function.
             If you don't specify anything, no activation is applied
             (ie. "linear" activation: a(x) = x).
-        weights: list of numpy arrays to set as initial weights.
+        weights: list of Numpy arrays to set as initial weights.
         border_mode: 'valid' or 'same'.
         subsample: tuple of length 3. Factor by which to subsample output.
             Also called strides elsewhere.
@@ -407,6 +424,10 @@ class Convolution3D(Layer):
             applied to the bias.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
 
     # Input shape
         5D tensor with shape:
@@ -424,9 +445,10 @@ class Convolution3D(Layer):
 
     def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3,
                  init='glorot_uniform', activation='linear', weights=None,
-                 border_mode='valid', subsample=(1, 1, 1), dim_ordering='th',
+                 border_mode='valid', subsample=(1, 1, 1), dim_ordering=K.image_dim_ordering(),
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
-                 W_constraint=None, b_constraint=None, **kwargs):
+                 W_constraint=None, b_constraint=None,
+                 bias=True, **kwargs):
         if K._BACKEND != 'theano':
             raise Exception(self.__class__.__name__ +
                             ' is currently only working with Theano backend.')
@@ -451,6 +473,7 @@ class Convolution3D(Layer):
         self.W_constraint = constraints.get(W_constraint)
         self.b_constraint = constraints.get(b_constraint)
 
+        self.bias = bias
         self.input_spec = [InputSpec(ndim=5)]
         self.initial_weights = weights
         super(Convolution3D, self).__init__(**kwargs)
@@ -471,15 +494,18 @@ class Convolution3D(Layer):
             raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
 
         self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
-        self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
-        self.trainable_weights = [self.W, self.b]
-        self.regularizers = []
+        if self.bias:
+            self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
+            self.trainable_weights = [self.W, self.b]
+        else:
+            self.trainable_weights = [self.W]
 
+        self.regularizers = []
         if self.W_regularizer:
             self.W_regularizer.set_param(self.W)
             self.regularizers.append(self.W_regularizer)
 
-        if self.b_regularizer:
+        if self.bias and self.b_regularizer:
             self.b_regularizer.set_param(self.b)
             self.regularizers.append(self.b_regularizer)
 
@@ -490,7 +516,7 @@ class Convolution3D(Layer):
         self.constraints = {}
         if self.W_constraint:
             self.constraints[self.W] = self.W_constraint
-        if self.b_constraint:
+        if self.bias and self.b_constraint:
             self.constraints[self.b] = self.b_constraint
 
         if self.initial_weights is not None:
@@ -525,36 +551,37 @@ class Convolution3D(Layer):
 
     def call(self, x, mask=None):
         input_shape = self.input_spec[0].shape
-        conv_out = K.conv3d(x, self.W, strides=self.subsample,
-                            border_mode=self.border_mode,
-                            dim_ordering=self.dim_ordering,
-                            volume_shape=input_shape,
-                            filter_shape=self.W_shape)
-
-        if self.dim_ordering == 'th':
-            output = conv_out + K.reshape(self.b, (1, self.nb_filter, 1, 1, 1))
-        elif self.dim_ordering == 'tf':
-            output = conv_out + K.reshape(self.b, (1, 1, 1, 1, self.nb_filter))
-        else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+        output = K.conv3d(x, self.W, strides=self.subsample,
+                          border_mode=self.border_mode,
+                          dim_ordering=self.dim_ordering,
+                          volume_shape=input_shape,
+                          filter_shape=self.W_shape)
+        if self.bias:
+            if self.dim_ordering == 'th':
+                output += K.reshape(self.b, (1, self.nb_filter, 1, 1, 1))
+            elif self.dim_ordering == 'tf':
+                output += K.reshape(self.b, (1, 1, 1, 1, self.nb_filter))
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
         output = self.activation(output)
         return output
 
     def get_config(self):
-        config = {"nb_filter": self.nb_filter,
-                  "kernel_dim1": self.kernel_dim1,
-                  "kernel_dim2": self.kernel_dim2,
-                  "kernel_dim3": self.kernel_dim3,
-                  "dim_ordering": self.dim_ordering,
-                  "init": self.init.__name__,
-                  "activation": self.activation.__name__,
-                  "border_mode": self.border_mode,
-                  "subsample": self.subsample,
-                  "W_regularizer": self.W_regularizer.get_config() if self.W_regularizer else None,
-                  "b_regularizer": self.b_regularizer.get_config() if self.b_regularizer else None,
-                  "activity_regularizer": self.activity_regularizer.get_config() if self.activity_regularizer else None,
-                  "W_constraint": self.W_constraint.get_config() if self.W_constraint else None,
-                  "b_constraint": self.b_constraint.get_config() if self.b_constraint else None}
+        config = {'nb_filter': self.nb_filter,
+                  'kernel_dim1': self.kernel_dim1,
+                  'kernel_dim2': self.kernel_dim2,
+                  'kernel_dim3': self.kernel_dim3,
+                  'dim_ordering': self.dim_ordering,
+                  'init': self.init.__name__,
+                  'activation': self.activation.__name__,
+                  'border_mode': self.border_mode,
+                  'subsample': self.subsample,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
+                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
+                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias}
         base_config = super(Convolution3D, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
@@ -665,7 +692,7 @@ class _Pooling2D(Layer):
     '''
 
     def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
-                 dim_ordering='th', **kwargs):
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
         super(_Pooling2D, self).__init__(**kwargs)
         self.pool_size = tuple(pool_size)
         if strides is None:
@@ -731,6 +758,9 @@ class MaxPooling2D(_Pooling2D):
             Note: 'same' will only work with TensorFlow for the time being.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
 
     # Input shape
         4D tensor with shape:
@@ -746,7 +776,7 @@ class MaxPooling2D(_Pooling2D):
     '''
 
     def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
-                 dim_ordering='th', **kwargs):
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
         super(MaxPooling2D, self).__init__(pool_size, strides, border_mode,
                                            dim_ordering, **kwargs)
 
@@ -769,6 +799,9 @@ class AveragePooling2D(_Pooling2D):
             Note: 'same' will only work with TensorFlow for the time being.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
 
     # Input shape
         4D tensor with shape:
@@ -784,7 +817,7 @@ class AveragePooling2D(_Pooling2D):
     '''
 
     def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
-                 dim_ordering='th', **kwargs):
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
         super(AveragePooling2D, self).__init__(pool_size, strides, border_mode,
                                                dim_ordering, **kwargs)
 
@@ -800,7 +833,7 @@ class _Pooling3D(Layer):
     '''
 
     def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
-                 dim_ordering='th', **kwargs):
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
         super(_Pooling3D, self).__init__(**kwargs)
         self.pool_size = tuple(pool_size)
         if strides is None:
@@ -871,6 +904,9 @@ class MaxPooling3D(_Pooling3D):
         border_mode: 'valid' or 'same'.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
 
     # Input shape
         5D tensor with shape:
@@ -886,7 +922,7 @@ class MaxPooling3D(_Pooling3D):
     '''
 
     def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
-                 dim_ordering='th', **kwargs):
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
         if K._BACKEND != 'theano':
             raise Exception(self.__class__.__name__ +
                             ' is currently only working with Theano backend.')
@@ -913,6 +949,9 @@ class AveragePooling3D(_Pooling3D):
         border_mode: 'valid' or 'same'.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
 
     # Input shape
         5D tensor with shape:
@@ -928,7 +967,7 @@ class AveragePooling3D(_Pooling3D):
     '''
 
     def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
-                 dim_ordering='th', **kwargs):
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
         if K._BACKEND != 'theano':
             raise Exception(self.__class__.__name__ +
                             ' is currently only working with Theano backend.')
@@ -982,6 +1021,9 @@ class UpSampling2D(Layer):
         dim_ordering: 'th' or 'tf'.
             In 'th' mode, the channels dimension (the depth)
             is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
 
     # Input shape
         4D tensor with shape:
@@ -996,7 +1038,7 @@ class UpSampling2D(Layer):
         `(samples, upsampled_rows, upsampled_cols, channels)` if dim_ordering='tf'.
     '''
 
-    def __init__(self, size=(2, 2), dim_ordering='th', **kwargs):
+    def __init__(self, size=(2, 2), dim_ordering=K.image_dim_ordering(), **kwargs):
         self.size = tuple(size)
         assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
         self.dim_ordering = dim_ordering
@@ -1038,6 +1080,9 @@ class UpSampling3D(Layer):
         dim_ordering: 'th' or 'tf'.
             In 'th' mode, the channels dimension (the depth)
             is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
 
     # Input shape
         5D tensor with shape:
@@ -1052,10 +1097,10 @@ class UpSampling3D(Layer):
         `(samples, upsampled_dim1, upsampled_dim2, upsampled_dim3, channels)` if dim_ordering='tf'.
     '''
 
-    def __init__(self, size=(2, 2, 2), dim_ordering='th', **kwargs):
+    def __init__(self, size=(2, 2, 2), dim_ordering=K.image_dim_ordering(), **kwargs):
         if K._BACKEND != 'theano':
             raise Exception(self.__class__.__name__ +
-                            ' is currently only working with Theano backend.') 
+                            ' is currently only working with Theano backend.')
         self.size = tuple(size)
         assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
         self.dim_ordering = dim_ordering
@@ -1130,6 +1175,12 @@ class ZeroPadding2D(Layer):
         padding: tuple of int (length 2)
             How many zeros to add at the beginning and end of
             the 2 padding dimensions (axis 3 and 4).
+        dim_ordering: 'th' or 'tf'.
+            In 'th' mode, the channels dimension (the depth)
+            is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
 
     # Input shape
         4D tensor with shape:
@@ -1140,7 +1191,7 @@ class ZeroPadding2D(Layer):
         (samples, depth, first_padded_axis, second_padded_axis)
     '''
 
-    def __init__(self, padding=(1, 1), dim_ordering='th', **kwargs):
+    def __init__(self, padding=(1, 1), dim_ordering=K.image_dim_ordering(), **kwargs):
         super(ZeroPadding2D, self).__init__(**kwargs)
         self.padding = tuple(padding)
         assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
@@ -1184,6 +1235,12 @@ class ZeroPadding3D(Layer):
         padding: tuple of int (length 3)
             How many zeros to add at the beginning and end of
             the 3 padding dimensions (axis 3, 4 and 5).
+        dim_ordering: 'th' or 'tf'.
+            In 'th' mode, the channels dimension (the depth)
+            is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
 
     # Input shape
         5D tensor with shape:
@@ -1194,7 +1251,7 @@ class ZeroPadding3D(Layer):
         (samples, depth, first_padded_axis, second_padded_axis, third_axis_to_pad)
     '''
 
-    def __init__(self, padding=(1, 1, 1), dim_ordering='th', **kwargs):
+    def __init__(self, padding=(1, 1, 1), dim_ordering=K.image_dim_ordering(), **kwargs):
         if K._BACKEND != 'theano':
             raise Exception(self.__class__.__name__ +
                             ' is currently only working with Theano backend.')

keras/layers/core.py

@@ -161,7 +161,7 @@ class Reshape(Layer):
         '''Find and replace a single missing dimension in an output shape
         given an input shape.
 
-        A near direct port of the internal numpy function _fix_unknown_dimension
+        A near direct port of the internal Numpy function _fix_unknown_dimension
         in numpy/core/src/multiarray/shape.c
 
         # Arguments
@@ -402,6 +402,8 @@ class Lambda(Layer):
     def __init__(self, function, output_shape=None, arguments={}, **kwargs):
         self.function = function
         self.arguments = arguments
+        self.supports_masking = True
+
         if output_shape is None:
             self._output_shape = None
         elif type(output_shape) in {tuple, list}:
@@ -460,9 +462,9 @@ class Lambda(Layer):
 
         if isinstance(self._output_shape, python_types.LambdaType):
             if py3:
-                output_shape = marshal.dumps(self._output_shape.__code__)
+                output_shape = marshal.dumps(self._output_shape.__code__).decode('raw_unicode_escape')
             else:
-                output_shape = marshal.dumps(self._output_shape.func_code)
+                output_shape = marshal.dumps(self._output_shape.func_code).decode('raw_unicode_escape')
             output_shape_type = 'lambda'
         elif callable(self._output_shape):
             output_shape = self._output_shape.__name__
@@ -494,7 +496,7 @@ class Lambda(Layer):
         if output_shape_type == 'function':
             output_shape = globals()[config['output_shape']]
         elif output_shape_type == 'lambda':
-            output_shape = marshal.loads(config['output_shape'])
+            output_shape = marshal.loads(config['output_shape'].encode('raw_unicode_escape'))
             output_shape = python_types.FunctionType(output_shape, globals())
         else:
             output_shape = config['output_shape']
@@ -511,12 +513,14 @@ class Dense(Layer):
 
     ```python
         # as first layer in a sequential model:
-        model = Sequential(Dense(32, input_dim=16))
+        model = Sequential()
+        model.add(Dense(32, input_dim=16))
         # now the model will take as input arrays of shape (*, 16)
         # and output arrays of shape (*, 32)
 
         # this is equivalent to the above:
-        model = Sequential(Dense(32, input_shape=(16,)))
+        model = Sequential()
+        model.add(Dense(32, input_shape=(16,)))
 
         # after the first layer, you don't need to specify
         # the size of the input anymore:
@@ -535,7 +539,7 @@ class Dense(Layer):
             or alternatively, elementwise Theano function.
             If you don't specify anything, no activation is applied
             (ie. "linear" activation: a(x) = x).
-        weights: list of numpy arrays to set as initial weights.
+        weights: list of Numpy arrays to set as initial weights.
             The list should have 2 elements, of shape `(input_dim, output_dim)`
             and (output_dim,) for weights and biases respectively.
         W_regularizer: instance of [WeightRegularizer](../regularizers.md)
@@ -548,6 +552,7 @@ class Dense(Layer):
             (eg. maxnorm, nonneg), applied to the main weights matrix.
         b_constraint: instance of the [constraints](../constraints.md) module,
             applied to the bias.
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
         input_dim: dimensionality of the input (integer).
             This argument (or alternatively, the keyword argument `input_shape`)
             is required when using this layer as the first layer in a model.
@@ -560,7 +565,8 @@ class Dense(Layer):
     '''
     def __init__(self, output_dim, init='glorot_uniform', activation='linear', weights=None,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
-                 W_constraint=None, b_constraint=None, input_dim=None, **kwargs):
+                 W_constraint=None, b_constraint=None,
+                 bias=True, input_dim=None, **kwargs):
         self.init = initializations.get(init)
         self.activation = activations.get(activation)
         self.output_dim = output_dim
@@ -573,6 +579,7 @@ class Dense(Layer):
         self.W_constraint = constraints.get(W_constraint)
         self.b_constraint = constraints.get(b_constraint)
 
+        self.bias = bias
         self.initial_weights = weights
         self.input_spec = [InputSpec(ndim=2)]
 
@@ -588,16 +595,19 @@ class Dense(Layer):
 
         self.W = self.init((input_dim, self.output_dim),
                            name='{}_W'.format(self.name))
-        self.b = K.zeros((self.output_dim,),
-                         name='{}_b'.format(self.name))
-        self.trainable_weights = [self.W, self.b]
+        if self.bias:
+            self.b = K.zeros((self.output_dim,),
+                             name='{}_b'.format(self.name))
+            self.trainable_weights = [self.W, self.b]
+        else:
+            self.trainable_weights = [self.W]
 
         self.regularizers = []
         if self.W_regularizer:
             self.W_regularizer.set_param(self.W)
             self.regularizers.append(self.W_regularizer)
 
-        if self.b_regularizer:
+        if self.bias and self.b_regularizer:
             self.b_regularizer.set_param(self.b)
             self.regularizers.append(self.b_regularizer)
 
@@ -608,7 +618,7 @@ class Dense(Layer):
         self.constraints = {}
         if self.W_constraint:
             self.constraints[self.W] = self.W_constraint
-        if self.b_constraint:
+        if self.bias and self.b_constraint:
             self.constraints[self.b] = self.b_constraint
 
         if self.initial_weights is not None:
@@ -616,7 +626,10 @@ class Dense(Layer):
             del self.initial_weights
 
     def call(self, x, mask=None):
-        return self.activation(K.dot(x, self.W) + self.b)
+        output = K.dot(x, self.W)
+        if self.bias:
+            output += self.b
+        return self.activation(output)
 
     def get_output_shape_for(self, input_shape):
         assert input_shape and len(input_shape) == 2
@@ -631,6 +644,7 @@ class Dense(Layer):
                   'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                   'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                   'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias,
                   'input_dim': self.input_dim}
         base_config = super(Dense, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -657,10 +671,10 @@ class ActivityRegularization(Layer):
         self.l1 = l1
         self.l2 = l2
 
+        super(ActivityRegularization, self).__init__(**kwargs)
         activity_regularizer = ActivityRegularizer(l1=l1, l2=l2)
         activity_regularizer.set_layer(self)
         self.regularizers = [activity_regularizer]
-        super(ActivityRegularization, self).__init__(**kwargs)
 
     def get_config(self):
         config = {'l1': self.l1,
@@ -690,12 +704,7 @@ class MaxoutDense(Layer):
             or alternatively, Theano function to use for weights
             initialization. This parameter is only relevant
             if you don't pass a `weights` argument.
-        activation: name of activation function to use
-            (see [activations](../activations.md)),
-            or alternatively, elementwise Theano function.
-            If you don't specify anything, no activation is applied
-            (ie. "linear" activation: a(x) = x).
-        weights: list of numpy arrays to set as initial weights.
+        weights: list of Numpy arrays to set as initial weights.
             The list should have 2 elements, of shape `(input_dim, output_dim)`
             and (output_dim,) for weights and biases respectively.
         W_regularizer: instance of [WeightRegularizer](../regularizers.md)
@@ -708,6 +717,7 @@ class MaxoutDense(Layer):
             (eg. maxnorm, nonneg), applied to the main weights matrix.
         b_constraint: instance of the [constraints](../constraints.md) module,
             applied to the bias.
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
         input_dim: dimensionality of the input (integer).
             This argument (or alternatively, the keyword argument `input_shape`)
             is required when using this layer as the first layer in a model.
@@ -724,7 +734,8 @@ class MaxoutDense(Layer):
     def __init__(self, output_dim, nb_feature=4,
                  init='glorot_uniform', weights=None,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
-                 W_constraint=None, b_constraint=None, input_dim=None, **kwargs):
+                 W_constraint=None, b_constraint=None,
+                 bias=True, input_dim=None, **kwargs):
         self.output_dim = output_dim
         self.nb_feature = nb_feature
         self.init = initializations.get(init)
@@ -736,6 +747,7 @@ class MaxoutDense(Layer):
         self.W_constraint = constraints.get(W_constraint)
         self.b_constraint = constraints.get(b_constraint)
 
+        self.bias = bias
         self.initial_weights = weights
         self.input_spec = [InputSpec(ndim=2)]
 
@@ -751,17 +763,19 @@ class MaxoutDense(Layer):
 
         self.W = self.init((self.nb_feature, input_dim, self.output_dim),
                            name='{}_W'.format(self.name))
-        self.b = K.zeros((self.nb_feature, self.output_dim),
-                         name='{}_b'.format(self.name))
+        if self.bias:
+            self.b = K.zeros((self.nb_feature, self.output_dim),
+                             name='{}_b'.format(self.name))
+            self.trainable_weights = [self.W, self.b]
+        else:
+            self.trainable_weights = [self.W]
 
-        self.trainable_weights = [self.W, self.b]
         self.regularizers = []
-
         if self.W_regularizer:
             self.W_regularizer.set_param(self.W)
             self.regularizers.append(self.W_regularizer)
 
-        if self.b_regularizer:
+        if self.bias and self.b_regularizer:
             self.b_regularizer.set_param(self.b)
             self.regularizers.append(self.b_regularizer)
 
@@ -772,7 +786,7 @@ class MaxoutDense(Layer):
         self.constraints = {}
         if self.W_constraint:
             self.constraints[self.W] = self.W_constraint
-        if self.b_constraint:
+        if self.bias and self.b_constraint:
             self.constraints[self.b] = self.b_constraint
 
         if self.initial_weights is not None:
@@ -785,7 +799,10 @@ class MaxoutDense(Layer):
 
     def call(self, x, mask=None):
         # no activation, this layer is only linear.
-        output = K.max(K.dot(x, self.W) + self.b, axis=1)
+        output = K.dot(x, self.W)
+        if self.bias:
+            output += self.b
+        output = K.max(output, axis=1)
         return output
 
     def get_config(self):
@@ -797,6 +814,7 @@ class MaxoutDense(Layer):
                   'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                   'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                   'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias,
                   'input_dim': self.input_dim}
         base_config = super(MaxoutDense, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -818,7 +836,7 @@ class Highway(Layer):
             or alternatively, elementwise Theano function.
             If you don't specify anything, no activation is applied
             (ie. "linear" activation: a(x) = x).
-        weights: list of numpy arrays to set as initial weights.
+        weights: list of Numpy arrays to set as initial weights.
             The list should have 2 elements, of shape `(input_dim, output_dim)`
             and (output_dim,) for weights and biases respectively.
         W_regularizer: instance of [WeightRegularizer](../regularizers.md)
@@ -831,6 +849,7 @@ class Highway(Layer):
             (eg. maxnorm, nonneg), applied to the main weights matrix.
         b_constraint: instance of the [constraints](../constraints.md) module,
             applied to the bias.
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
         input_dim: dimensionality of the input (integer).
             This argument (or alternatively, the keyword argument `input_shape`)
             is required when using this layer as the first layer in a model.
@@ -847,7 +866,8 @@ class Highway(Layer):
     def __init__(self, init='glorot_uniform', transform_bias=-2,
                  activation='linear', weights=None,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
-                 W_constraint=None, b_constraint=None, input_dim=None, **kwargs):
+                 W_constraint=None, b_constraint=None,
+                 bias=True, input_dim=None, **kwargs):
         self.init = initializations.get(init)
         self.transform_bias = transform_bias
         self.activation = activations.get(activation)
@@ -859,6 +879,7 @@ class Highway(Layer):
         self.W_constraint = constraints.get(W_constraint)
         self.b_constraint = constraints.get(b_constraint)
 
+        self.bias = bias
         self.initial_weights = weights
         self.input_spec = [InputSpec(ndim=2)]
 
@@ -877,19 +898,21 @@ class Highway(Layer):
         self.W_carry = self.init((input_dim, input_dim),
                                  name='{}_W_carry'.format(self.name))
 
-        self.b = K.zeros((input_dim,), name='{}_b'.format(self.name))
-        # initialize with a vector of values `transform_bias`
-        self.b_carry = K.variable(np.ones((input_dim,)) * self.transform_bias,
-                                  name='{}_b_carry'.format(self.name))
-
-        self.trainable_weights = [self.W, self.b, self.W_carry, self.b_carry]
+        if self.bias:
+            self.b = K.zeros((input_dim,), name='{}_b'.format(self.name))
+            # initialize with a vector of values `transform_bias`
+            self.b_carry = K.variable(np.ones((input_dim,)) * self.transform_bias,
+                                      name='{}_b_carry'.format(self.name))
+            self.trainable_weights = [self.W, self.b, self.W_carry, self.b_carry]
+        else:
+            self.trainable_weights = [self.W, self.W_carry]
 
         self.regularizers = []
         if self.W_regularizer:
             self.W_regularizer.set_param(self.W)
             self.regularizers.append(self.W_regularizer)
 
-        if self.b_regularizer:
+        if self.bias and self.b_regularizer:
             self.b_regularizer.set_param(self.b)
             self.regularizers.append(self.b_regularizer)
 
@@ -900,7 +923,7 @@ class Highway(Layer):
         self.constraints = {}
         if self.W_constraint:
             self.constraints[self.W] = self.W_constraint
-        if self.b_constraint:
+        if self.bias and self.b_constraint:
             self.constraints[self.b] = self.b_constraint
 
         if self.initial_weights is not None:
@@ -908,8 +931,14 @@ class Highway(Layer):
             del self.initial_weights
 
     def call(self, x, mask=None):
-        transform_weight = activations.sigmoid(K.dot(x, self.W_carry) + self.b_carry)
-        act = self.activation(K.dot(x, self.W) + self.b)
+        y = K.dot(x, self.W_carry)
+        if self.bias:
+            y += self.b_carry
+        transform_weight = activations.sigmoid(y)
+        y = K.dot(x, self.W)
+        if self.bias:
+            y += self.b
+        act = self.activation(y)
         act *= transform_weight
         output = act + (1 - transform_weight) * x
         return output
@@ -923,6 +952,7 @@ class Highway(Layer):
                   'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                   'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                   'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias,
                   'input_dim': self.input_dim}
         base_config = super(Highway, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -939,8 +969,10 @@ class TimeDistributedDense(Layer):
 
     # Input shape
         3D tensor with shape `(nb_sample, time_dimension, input_dim)`.
+
     # Output shape
         3D tensor with shape `(nb_sample, time_dimension, output_dim)`.
+
     # Arguments
         output_dim: int > 0.
         init: name of initialization function for the weights of the layer
@@ -953,7 +985,7 @@ class TimeDistributedDense(Layer):
             or alternatively, elementwise Theano function.
             If you don't specify anything, no activation is applied
             (ie. "linear" activation: a(x) = x).
-        weights: list of numpy arrays to set as initial weights.
+        weights: list of Numpy arrays to set as initial weights.
             The list should have 2 elements, of shape `(input_dim, output_dim)`
             and (output_dim,) for weights and biases respectively.
         W_regularizer: instance of [WeightRegularizer](../regularizers.md)
@@ -966,16 +998,19 @@ class TimeDistributedDense(Layer):
             (eg. maxnorm, nonneg), applied to the main weights matrix.
         b_constraint: instance of the [constraints](../constraints.md) module,
             applied to the bias.
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
         input_dim: dimensionality of the input (integer).
             This argument (or alternatively, the keyword argument `input_shape`)
             is required when using this layer as the first layer in a model.
+        input_length: length of inputs sequences
+            (integer, or None for variable-length sequences).
     '''
 
     def __init__(self, output_dim,
                  init='glorot_uniform', activation='linear', weights=None,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
-                 input_dim=None, input_length=None, **kwargs):
+                 bias=True, input_dim=None, input_length=None, **kwargs):
         warnings.warn('TimeDistributedDense is deprecated, '
                       'please use TimeDistributed(Dense(...)) instead.')
         self.output_dim = output_dim
@@ -989,6 +1024,7 @@ class TimeDistributedDense(Layer):
         self.W_constraint = constraints.get(W_constraint)
         self.b_constraint = constraints.get(b_constraint)
 
+        self.bias = bias
         self.initial_weights = weights
         self.input_spec = [InputSpec(ndim=3)]
         self.supports_masking = True
@@ -1006,17 +1042,17 @@ class TimeDistributedDense(Layer):
 
         self.W = self.init((input_dim, self.output_dim),
                            name='{}_W'.format(self.name))
-        self.b = K.zeros((self.output_dim,),
-                         name='{}_b'.format(self.name))
-
-        self.trainable_weights = [self.W, self.b]
+        if self.bias:
+            self.b = K.zeros((self.output_dim,),
+                             name='{}_b'.format(self.name))
+            self.trainable_weights = [self.W, self.b]
         self.regularizers = []
 
         if self.W_regularizer:
             self.W_regularizer.set_param(self.W)
             self.regularizers.append(self.W_regularizer)
 
-        if self.b_regularizer:
+        if self.bias and self.b_regularizer:
             self.b_regularizer.set_param(self.b)
             self.regularizers.append(self.b_regularizer)
 
@@ -1027,7 +1063,7 @@ class TimeDistributedDense(Layer):
         self.constraints = {}
         if self.W_constraint:
             self.constraints[self.W] = self.W_constraint
-        if self.b_constraint:
+        if self.bias and self.b_constraint:
             self.constraints[self.b] = self.b_constraint
 
         if self.initial_weights is not None:
@@ -1057,7 +1093,9 @@ class TimeDistributedDense(Layer):
 
         # Squash samples and timesteps into a single axis
         x = K.reshape(x, (-1, input_shape[-1]))  # (samples * timesteps, input_dim)
-        y = K.dot(x, self.W) + self.b  # (samples * timesteps, output_dim)
+        y = K.dot(x, self.W)  # (samples * timesteps, output_dim)
+        if self.bias:
+            y += self.b
         # We have to reshape Y to (samples, timesteps, output_dim)
         y = K.reshape(y, (-1, input_length, self.output_dim))  # (samples, timesteps, output_dim)
         y = self.activation(y)
@@ -1072,6 +1110,7 @@ class TimeDistributedDense(Layer):
                   'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
                   'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
                   'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias,
                   'input_dim': self.input_dim,
                   'input_length': self.input_length}
         base_config = super(TimeDistributedDense, self).get_config()

keras/layers/embeddings.py

@@ -17,7 +17,7 @@ class Embedding(Layer):
       model = Sequential()
       model.add(Embedding(1000, 64, input_length=10))
       # the model will take as input an integer matrix of size (batch, input_length).
-      # the largest integer (i.e. word index) in the input should be no larger than 1000 (vocabulary size).
+      # the largest integer (i.e. word index) in the input should be no larger than 999 (vocabulary size).
       # now model.output_shape == (None, 10, 64), where None is the batch dimension.
 
       input_array = np.random.randint(1000, size=(32, 10))
@@ -28,14 +28,14 @@ class Embedding(Layer):
     ```
 
     # Arguments
-      input_dim: int >= 0. Size of the vocabulary, ie.
+      input_dim: int > 0. Size of the vocabulary, ie.
           1 + maximum integer index occurring in the input data.
       output_dim: int >= 0. Dimension of the dense embedding.
       init: name of initialization function for the weights
           of the layer (see: [initializations](../initializations.md)),
           or alternatively, Theano function to use for weights initialization.
           This parameter is only relevant if you don't pass a `weights` argument.
-      weights: list of numpy arrays to set as initial weights.
+      weights: list of Numpy arrays to set as initial weights.
           The list should have 1 element, of shape `(input_dim, output_dim)`.
       W_regularizer: instance of the [regularizers](../regularizers.md) module
         (eg. L1 or L2 regularization), applied to the embedding matrix.
@@ -46,6 +46,8 @@ class Embedding(Layer):
           This is useful for [recurrent layers](recurrent.md) which may take
           variable length input. If this is `True` then all subsequent layers
           in the model need to support masking or an exception will be raised.
+          If mask_zero is set to True, as a consequence, index 0 cannot be
+          used in the vocabulary (input_dim should equal |vocabulary| + 2).
       input_length: Length of input sequences, when it is constant.
           This argument is required if you are going to connect
           `Flatten` then `Dense` layers upstream

keras/layers/noise.py

@@ -4,7 +4,7 @@ from .. import backend as K
 
 
 class GaussianNoise(Layer):
-    '''Apply to the input an additive zero-centred gaussian noise with
+    '''Apply to the input an additive zero-centered Gaussian noise with
     standard deviation `sigma`. This is useful to mitigate overfitting
     (you could see it as a kind of random data augmentation).
     Gaussian Noise (GS) is a natural choice as corruption process
@@ -42,7 +42,7 @@ class GaussianNoise(Layer):
 
 
 class GaussianDropout(Layer):
-    '''Apply to the input an multiplicative one-centred gaussian noise
+    '''Apply to the input an multiplicative one-centered Gaussian noise
     with standard deviation `sqrt(p/(1-p))`.
 
     As it is a regularization layer, it is only active at training time.

keras/layers/normalization.py

@@ -10,7 +10,7 @@ class BatchNormalization(Layer):
 
     # Arguments
         epsilon: small float > 0. Fuzz parameter.
-        mode: integer, 0 or 1.
+        mode: integer, 0, 1 or 2.
             - 0: feature-wise normalization.
                 Each feature map in the input will
                 be normalized separately. The axis on which
@@ -19,7 +19,13 @@ class BatchNormalization(Layer):
                 using Theano conventions (samples, channels, rows, cols)
                 then you should set `axis` to `1` to normalize along
                 the channels axis.
+                During training we use per-batch statistics to normalize
+                the data, and during testing we use running averages
+                computed during the training phase.
             - 1: sample-wise normalization. This mode assumes a 2D input.
+            - 2: feature-wise normalization, like mode 0, but
+                using per-batch statistics to normalize the data during both
+                testing and training.
         axis: integer, axis along which to normalize in mode 0. For instance,
             if your input tensor has shape (samples, channels, rows, cols),
             set axis to 1 to normalize per feature map (channels axis).
@@ -27,7 +33,7 @@ class BatchNormalization(Layer):
             exponential average of the mean and standard deviation
             of the data, for feature-wise normalization.
         weights: Initialization weights.
-            List of 2 numpy arrays, with shapes:
+            List of 2 Numpy arrays, with shapes:
             `[(input_shape,), (input_shape,)]`
         beta_init: name of initialization function for shift parameter
             (see [initializations](../initializations.md)), or alternatively,
@@ -47,7 +53,7 @@ class BatchNormalization(Layer):
         Same shape as input.
 
     # References
-        - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](http://arxiv.org/pdf/1502.03167v3.pdf)
+        - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](http://jmlr.org/proceedings/papers/v37/ioffe15.html)
     '''
     def __init__(self, epsilon=1e-6, mode=0, axis=-1, momentum=0.9,
                  weights=None, beta_init='zero', gamma_init='one', **kwargs):
@@ -58,7 +64,8 @@ class BatchNormalization(Layer):
         self.axis = axis
         self.momentum = momentum
         self.initial_weights = weights
-        self.uses_learning_phase = True
+        if self.mode == 0:
+            self.uses_learning_phase = True
         super(BatchNormalization, self).__init__(**kwargs)
 
     def build(self, input_shape):
@@ -78,9 +85,12 @@ class BatchNormalization(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
+        self.called_with = None
 
     def call(self, x, mask=None):
-        if self.mode == 0:
+        if self.mode == 0 or self.mode == 2:
+            assert self.built, 'Layer must be built before being called'
             input_shape = self.input_spec[0].shape
 
             reduction_axes = list(range(len(input_shape)))
@@ -94,25 +104,42 @@ class BatchNormalization(Layer):
             std = K.mean(K.square(x - brodcast_mean) + self.epsilon, axis=reduction_axes)
             std = K.sqrt(std)
             brodcast_std = K.reshape(std, broadcast_shape)
-            mean_update = self.momentum * self.running_mean + (1-self.momentum) * mean
-            std_update = self.momentum * self.running_std + (1-self.momentum) * std
-            self.updates = [(self.running_mean, mean_update),
-                            (self.running_std, std_update)]
-            x_normed = (x - brodcast_mean) / (brodcast_std + self.epsilon)
+            mean_update = self.momentum * self.running_mean + (1 - self.momentum) * mean
+            std_update = self.momentum * self.running_std + (1 - self.momentum) * std
 
-            # case: test mode (uses running averages)
-            brodcast_running_mean = K.reshape(self.running_mean, broadcast_shape)
-            brodcast_running_std = K.reshape(self.running_std, broadcast_shape)
-            x_normed_running = ((x - brodcast_running_mean) / (brodcast_running_std + self.epsilon))
+            if self.mode == 2:
+                x_normed = (x - brodcast_mean) / (brodcast_std + self.epsilon)
+                out = K.reshape(self.gamma, broadcast_shape) * x_normed + K.reshape(self.beta, broadcast_shape)
+            else:
+                # mode 0
+                if self.called_with not in {None, x}:
+                    raise Exception('You are attempting to share a '
+                                    'same `BatchNormalization` layer across '
+                                    'different data flows. '
+                                    'This is not possible. '
+                                    'You should use `mode=2` in '
+                                    '`BatchNormalization`, which has '
+                                    'a similar behavior but is shareable '
+                                    '(see docs for a description of '
+                                    'the behavior).')
+                self.called_with = x
+                self.updates = [(self.running_mean, mean_update),
+                                (self.running_std, std_update)]
+                x_normed = (x - brodcast_mean) / (brodcast_std + self.epsilon)
 
-            # pick the normalized form of x corresponding to the training phase
-            x_normed = K.in_train_phase(x_normed, x_normed_running)
-            out = K.reshape(self.gamma, broadcast_shape) * x_normed + K.reshape(self.beta, broadcast_shape)
+                # case: test mode (uses running averages)
+                brodcast_running_mean = K.reshape(self.running_mean, broadcast_shape)
+                brodcast_running_std = K.reshape(self.running_std, broadcast_shape)
+                x_normed_running = ((x - brodcast_running_mean) / (brodcast_running_std + self.epsilon))
+
+                # pick the normalized form of x corresponding to the training phase
+                x_normed = K.in_train_phase(x_normed, x_normed_running)
+                out = K.reshape(self.gamma, broadcast_shape) * x_normed + K.reshape(self.beta, broadcast_shape)
 
         elif self.mode == 1:
             # sample-wise normalization
             m = K.mean(x, axis=-1, keepdims=True)
-            std = K.std(x, axis=-1, keepdims=True)
+            std = K.sqrt(K.var(x, axis=-1, keepdims=True) + self.epsilon)
             x_normed = (x - m) / (std + self.epsilon)
             out = self.gamma * x_normed + self.beta
         return out

keras/layers/recurrent.py

@@ -54,7 +54,7 @@ class Recurrent(Layer):
         # as the first layer in a Sequential model
         model = Sequential()
         model.add(LSTM(32, input_shape=(10, 64)))
-        # now model.output_shape == (None, 10, 32)
+        # now model.output_shape == (None, 32)
         # note: `None` is the batch dimension.
 
         # the following is identical:
@@ -66,7 +66,7 @@ class Recurrent(Layer):
     ```
 
     # Arguments
-        weights: list of numpy arrays to set as initial weights.
+        weights: list of Numpy arrays to set as initial weights.
             The list should have 3 elements, of shapes:
             `[(input_dim, output_dim), (output_dim, output_dim), (output_dim,)]`.
         return_sequences: Boolean. Whether to return the last output
@@ -81,12 +81,18 @@ class Recurrent(Layer):
             is always unrolled, so this argument does not do anything.
             Unrolling can speed-up a RNN, although it tends to be more memory-intensive.
             Unrolling is only suitable for short sequences.
-        consume_less: one of "cpu", "mem". If set to "cpu", the RNN will use
+        consume_less: one of "cpu", "mem", or "gpu" (LSTM/GRU only).
+            If set to "cpu", the RNN will use
             an implementation that uses fewer, larger matrix products,
-            thus running faster (at least on CPU) but consuming more memory.
+            thus running faster on CPU but consuming more memory.
             If set to "mem", the RNN will use more matrix products,
             but smaller ones, thus running slower (may actually be faster on GPU)
             while consuming less memory.
+            If set to "gpu" (LSTM/GRU only), the RNN will combine the input gate,
+            the forget gate and the output gate into a single matrix,
+            enabling more time-efficient parallelization on the GPU. Note: RNN
+            dropout must be shared for all gates, resulting in a slightly
+            reduced regularization.
         input_dim: dimensionality of the input (integer).
             This argument (or alternatively, the keyword argument `input_shape`)
             is required when using this layer as the first layer in a model.
@@ -383,15 +389,15 @@ class SimpleRNN(Recurrent):
         return constants
 
     def get_config(self):
-        config = {"output_dim": self.output_dim,
-                  "init": self.init.__name__,
-                  "inner_init": self.inner_init.__name__,
-                  "activation": self.activation.__name__,
-                  "W_regularizer": self.W_regularizer.get_config() if self.W_regularizer else None,
-                  "U_regularizer": self.U_regularizer.get_config() if self.U_regularizer else None,
-                  "b_regularizer": self.b_regularizer.get_config() if self.b_regularizer else None,
-                  "dropout_W": self.dropout_W,
-                  "dropout_U": self.dropout_U}
+        config = {'output_dim': self.output_dim,
+                  'init': self.init.__name__,
+                  'inner_init': self.inner_init.__name__,
+                  'activation': self.activation.__name__,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'U_regularizer': self.U_regularizer.get_config() if self.U_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'dropout_W': self.dropout_W,
+                  'dropout_U': self.dropout_U}
         base_config = super(SimpleRNN, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
@@ -444,53 +450,66 @@ class GRU(Recurrent):
 
     def build(self, input_shape):
         self.input_spec = [InputSpec(shape=input_shape)]
-        input_dim = input_shape[2]
-        self.input_dim = input_dim
+        self.input_dim = input_shape[2]
 
-        self.W_z = self.init((input_dim, self.output_dim),
-                             name='{}_W_z'.format(self.name))
-        self.U_z = self.inner_init((self.output_dim, self.output_dim),
-                                   name='{}_U_z'.format(self.name))
-        self.b_z = K.zeros((self.output_dim,), name='{}_b_z'.format(self.name))
-
-        self.W_r = self.init((input_dim, self.output_dim),
-                             name='{}_W_r'.format(self.name))
-        self.U_r = self.inner_init((self.output_dim, self.output_dim),
-                                   name='{}_U_r'.format(self.name))
-        self.b_r = K.zeros((self.output_dim,), name='{}_b_r'.format(self.name))
-
-        self.W_h = self.init((input_dim, self.output_dim),
-                             name='{}_W_h'.format(self.name))
-        self.U_h = self.inner_init((self.output_dim, self.output_dim),
-                                   name='{}_U_h'.format(self.name))
-        self.b_h = K.zeros((self.output_dim,), name='{}_b_h'.format(self.name))
-
-        self.regularizers = []
-        if self.W_regularizer:
-            self.W_regularizer.set_param(K.concatenate([self.W_z,
-                                                        self.W_r,
-                                                        self.W_h]))
-            self.regularizers.append(self.W_regularizer)
-        if self.U_regularizer:
-            self.U_regularizer.set_param(K.concatenate([self.U_z,
-                                                        self.U_r,
-                                                        self.U_h]))
-            self.regularizers.append(self.U_regularizer)
-        if self.b_regularizer:
-            self.b_regularizer.set_param(K.concatenate([self.b_z,
-                                                        self.b_r,
-                                                        self.b_h]))
-            self.regularizers.append(self.b_regularizer)
-
-        self.trainable_weights = [self.W_z, self.U_z, self.b_z,
-                                  self.W_r, self.U_r, self.b_r,
-                                  self.W_h, self.U_h, self.b_h]
         if self.stateful:
             self.reset_states()
         else:
             # initial states: all-zero tensor of shape (output_dim)
             self.states = [None]
 
+        if self.consume_less == 'gpu':
+
+            self.W = self.init((self.input_dim, 3 * self.output_dim),
+                               name='{}_W'.format(self.name))
+            self.U = self.inner_init((self.output_dim, 3 * self.output_dim),
+                                     name='{}_U'.format(self.name))
+
+            self.b = K.variable(np.hstack((np.zeros(self.output_dim),
+                                           np.zeros(self.output_dim),
+                                           np.zeros(self.output_dim))),
+                                name='{}_b'.format(self.name))
+
+            self.trainable_weights = [self.W, self.U, self.b]
+        else:
+
+            self.W_z = self.init((self.input_dim, self.output_dim),
+                                 name='{}_W_z'.format(self.name))
+            self.U_z = self.inner_init((self.output_dim, self.output_dim),
+                                       name='{}_U_z'.format(self.name))
+            self.b_z = K.zeros((self.output_dim,), name='{}_b_z'.format(self.name))
+
+            self.W_r = self.init((self.input_dim, self.output_dim),
+                                 name='{}_W_r'.format(self.name))
+            self.U_r = self.inner_init((self.output_dim, self.output_dim),
+                                       name='{}_U_r'.format(self.name))
+            self.b_r = K.zeros((self.output_dim,), name='{}_b_r'.format(self.name))
+
+            self.W_h = self.init((self.input_dim, self.output_dim),
+                                 name='{}_W_h'.format(self.name))
+            self.U_h = self.inner_init((self.output_dim, self.output_dim),
+                                       name='{}_U_h'.format(self.name))
+            self.b_h = K.zeros((self.output_dim,), name='{}_b_h'.format(self.name))
+
+            self.trainable_weights = [self.W_z, self.U_z, self.b_z,
+                                      self.W_r, self.U_r, self.b_r,
+                                      self.W_h, self.U_h, self.b_h]
+
+            self.W = K.concatenate([self.W_z, self.W_r, self.W_h])
+            self.U = K.concatenate([self.U_z, self.U_r, self.U_h])
+            self.b = K.concatenate([self.b_z, self.b_r, self.b_h])
+
+        self.regularizers = []
+        if self.W_regularizer:
+            self.W_regularizer.set_param(self.W)
+            self.regularizers.append(self.W_regularizer)
+        if self.U_regularizer:
+            self.U_regularizer.set_param(self.U)
+            self.regularizers.append(self.U_regularizer)
+        if self.b_regularizer:
+            self.b_regularizer.set_param(self.b)
+            self.regularizers.append(self.b_regularizer)
+
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
@@ -528,19 +547,37 @@ class GRU(Recurrent):
         B_U = states[1]  # dropout matrices for recurrent units
         B_W = states[2]
 
-        if self.consume_less == 'cpu':
-            x_z = x[:, :self.output_dim]
-            x_r = x[:, self.output_dim: 2 * self.output_dim]
-            x_h = x[:, 2 * self.output_dim:]
+        if self.consume_less == 'gpu':
+
+            matrix_x = K.dot(x * B_W[0], self.W) + self.b
+            matrix_inner = K.dot(h_tm1 * B_U[0], self.U[:, :2 * self.output_dim])
+
+            x_z = matrix_x[:, :self.output_dim]
+            x_r = matrix_x[:, self.output_dim: 2 * self.output_dim]
+            inner_z = matrix_inner[:, :self.output_dim]
+            inner_r = matrix_inner[:, self.output_dim: 2 * self.output_dim]
+
+            z = self.inner_activation(x_z + inner_z)
+            r = self.inner_activation(x_r + inner_r)
+
+            x_h = matrix_x[:, 2 * self.output_dim:]
+            inner_h = K.dot(r * h_tm1 * B_U[0], self.U[:, 2 * self.output_dim:])
+            hh = self.activation(x_h + inner_h)
         else:
-            x_z = K.dot(x * B_W[0], self.W_z) + self.b_z
-            x_r = K.dot(x * B_W[1], self.W_r) + self.b_r
-            x_h = K.dot(x * B_W[2], self.W_h) + self.b_h
+            if self.consume_less == 'cpu':
+                x_z = x[:, :self.output_dim]
+                x_r = x[:, self.output_dim: 2 * self.output_dim]
+                x_h = x[:, 2 * self.output_dim:]
+            elif self.consume_less == 'mem':
+                x_z = K.dot(x * B_W[0], self.W_z) + self.b_z
+                x_r = K.dot(x * B_W[1], self.W_r) + self.b_r
+                x_h = K.dot(x * B_W[2], self.W_h) + self.b_h
+            else:
+                raise Exception('Unknown `consume_less` mode.')
+            z = self.inner_activation(x_z + K.dot(h_tm1 * B_U[0], self.U_z))
+            r = self.inner_activation(x_r + K.dot(h_tm1 * B_U[1], self.U_r))
 
-        z = self.inner_activation(x_z + K.dot(h_tm1 * B_U[0], self.U_z))
-        r = self.inner_activation(x_r + K.dot(h_tm1 * B_U[1], self.U_r))
-
-        hh = self.activation(x_h + K.dot(r * h_tm1 * B_U[2], self.U_h))
+            hh = self.activation(x_h + K.dot(r * h_tm1 * B_U[2], self.U_h))
         h = z * h_tm1 + (1 - z) * hh
         return h, [h]
 
@@ -552,7 +589,7 @@ class GRU(Recurrent):
             B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones) for _ in range(3)]
             constants.append(B_U)
         else:
-            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
+            constants.append([K.cast_to_floatx(1.) for _ in range(3)])
 
         if 0 < self.dropout_W < 1:
             input_shape = self.input_spec[0].shape
@@ -562,20 +599,20 @@ class GRU(Recurrent):
             B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(3)]
             constants.append(B_W)
         else:
-            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
+            constants.append([K.cast_to_floatx(1.) for _ in range(3)])
         return constants
 
     def get_config(self):
-        config = {"output_dim": self.output_dim,
-                  "init": self.init.__name__,
-                  "inner_init": self.inner_init.__name__,
-                  "activation": self.activation.__name__,
-                  "inner_activation": self.inner_activation.__name__,
-                  "W_regularizer": self.W_regularizer.get_config() if self.W_regularizer else None,
-                  "U_regularizer": self.U_regularizer.get_config() if self.U_regularizer else None,
-                  "b_regularizer": self.b_regularizer.get_config() if self.b_regularizer else None,
-                  "dropout_W": self.dropout_W,
-                  "dropout_U": self.dropout_U}
+        config = {'output_dim': self.output_dim,
+                  'init': self.init.__name__,
+                  'inner_init': self.inner_init.__name__,
+                  'activation': self.activation.__name__,
+                  'inner_activation': self.inner_activation.__name__,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'U_regularizer': self.U_regularizer.get_config() if self.U_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'dropout_W': self.dropout_W,
+                  'dropout_U': self.dropout_U}
         base_config = super(GRU, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
@@ -637,8 +674,7 @@ class LSTM(Recurrent):
 
     def build(self, input_shape):
         self.input_spec = [InputSpec(shape=input_shape)]
-        input_dim = input_shape[2]
-        self.input_dim = input_dim
+        self.input_dim = input_shape[2]
 
         if self.stateful:
             self.reset_states()
@@ -646,56 +682,64 @@ class LSTM(Recurrent):
             # initial states: 2 all-zero tensors of shape (output_dim)
             self.states = [None, None]
 
-        self.W_i = self.init((input_dim, self.output_dim),
-                             name='{}_W_i'.format(self.name))
-        self.U_i = self.inner_init((self.output_dim, self.output_dim),
-                                   name='{}_U_i'.format(self.name))
-        self.b_i = K.zeros((self.output_dim,), name='{}_b_i'.format(self.name))
+        if self.consume_less == 'gpu':
+            self.W = self.init((self.input_dim, 4 * self.output_dim),
+                               name='{}_W'.format(self.name))
+            self.U = self.inner_init((self.output_dim, 4 * self.output_dim),
+                                     name='{}_U'.format(self.name))
 
-        self.W_f = self.init((input_dim, self.output_dim),
-                             name='{}_W_f'.format(self.name))
-        self.U_f = self.inner_init((self.output_dim, self.output_dim),
-                                   name='{}_U_f'.format(self.name))
-        self.b_f = self.forget_bias_init((self.output_dim,),
-                                         name='{}_b_f'.format(self.name))
+            self.b = K.variable(np.hstack((np.zeros(self.output_dim),
+                                           K.get_value(self.forget_bias_init(self.output_dim)),
+                                           np.zeros(self.output_dim),
+                                           np.zeros(self.output_dim))),
+                                name='{}_b'.format(self.name))
+            self.trainable_weights = [self.W, self.U, self.b]
+        else:
+            self.W_i = self.init((self.input_dim, self.output_dim),
+                                 name='{}_W_i'.format(self.name))
+            self.U_i = self.inner_init((self.output_dim, self.output_dim),
+                                       name='{}_U_i'.format(self.name))
+            self.b_i = K.zeros((self.output_dim,), name='{}_b_i'.format(self.name))
 
-        self.W_c = self.init((input_dim, self.output_dim),
-                             name='{}_W_c'.format(self.name))
-        self.U_c = self.inner_init((self.output_dim, self.output_dim),
-                                   name='{}_U_c'.format(self.name))
-        self.b_c = K.zeros((self.output_dim,), name='{}_b_c'.format(self.name))
+            self.W_f = self.init((self.input_dim, self.output_dim),
+                                 name='{}_W_f'.format(self.name))
+            self.U_f = self.inner_init((self.output_dim, self.output_dim),
+                                       name='{}_U_f'.format(self.name))
+            self.b_f = self.forget_bias_init((self.output_dim,),
+                                             name='{}_b_f'.format(self.name))
 
-        self.W_o = self.init((input_dim, self.output_dim),
-                             name='{}_W_o'.format(self.name))
-        self.U_o = self.inner_init((self.output_dim, self.output_dim),
-                                   name='{}_U_o'.format(self.name))
-        self.b_o = K.zeros((self.output_dim,), name='{}_b_o'.format(self.name))
+            self.W_c = self.init((self.input_dim, self.output_dim),
+                                 name='{}_W_c'.format(self.name))
+            self.U_c = self.inner_init((self.output_dim, self.output_dim),
+                                       name='{}_U_c'.format(self.name))
+            self.b_c = K.zeros((self.output_dim,), name='{}_b_c'.format(self.name))
+
+            self.W_o = self.init((self.input_dim, self.output_dim),
+                                 name='{}_W_o'.format(self.name))
+            self.U_o = self.inner_init((self.output_dim, self.output_dim),
+                                       name='{}_U_o'.format(self.name))
+            self.b_o = K.zeros((self.output_dim,), name='{}_b_o'.format(self.name))
+
+            self.trainable_weights = [self.W_i, self.U_i, self.b_i,
+                                      self.W_c, self.U_c, self.b_c,
+                                      self.W_f, self.U_f, self.b_f,
+                                      self.W_o, self.U_o, self.b_o]
+
+            self.W = K.concatenate([self.W_i, self.W_f, self.W_c, self.W_o])
+            self.U = K.concatenate([self.U_i, self.U_f, self.U_c, self.U_o])
+            self.b = K.concatenate([self.b_i, self.b_f, self.b_c, self.b_o])
 
         self.regularizers = []
         if self.W_regularizer:
-            self.W_regularizer.set_param(K.concatenate([self.W_i,
-                                                        self.W_f,
-                                                        self.W_c,
-                                                        self.W_o]))
+            self.W_regularizer.set_param(self.W)
             self.regularizers.append(self.W_regularizer)
         if self.U_regularizer:
-            self.U_regularizer.set_param(K.concatenate([self.U_i,
-                                                        self.U_f,
-                                                        self.U_c,
-                                                        self.U_o]))
+            self.U_regularizer.set_param(self.U)
             self.regularizers.append(self.U_regularizer)
         if self.b_regularizer:
-            self.b_regularizer.set_param(K.concatenate([self.b_i,
-                                                        self.b_f,
-                                                        self.b_c,
-                                                        self.b_o]))
+            self.b_regularizer.set_param(self.b)
             self.regularizers.append(self.b_regularizer)
 
-        self.trainable_weights = [self.W_i, self.U_i, self.b_i,
-                                  self.W_c, self.U_c, self.b_c,
-                                  self.W_f, self.U_f, self.b_f,
-                                  self.W_o, self.U_o, self.b_o]
-
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
@@ -743,21 +787,36 @@ class LSTM(Recurrent):
         B_U = states[2]
         B_W = states[3]
 
-        if self.consume_less == 'cpu':
-            x_i = x[:, :self.output_dim]
-            x_f = x[:, self.output_dim: 2 * self.output_dim]
-            x_c = x[:, 2 * self.output_dim: 3 * self.output_dim]
-            x_o = x[:, 3 * self.output_dim:]
-        else:
-            x_i = K.dot(x * B_W[0], self.W_i) + self.b_i
-            x_f = K.dot(x * B_W[1], self.W_f) + self.b_f
-            x_c = K.dot(x * B_W[2], self.W_c) + self.b_c
-            x_o = K.dot(x * B_W[3], self.W_o) + self.b_o
+        if self.consume_less == 'gpu':
+            z = K.dot(x * B_W[0], self.W) + K.dot(h_tm1 * B_U[0], self.U) + self.b
 
-        i = self.inner_activation(x_i + K.dot(h_tm1 * B_U[0], self.U_i))
-        f = self.inner_activation(x_f + K.dot(h_tm1 * B_U[1], self.U_f))
-        c = f * c_tm1 + i * self.activation(x_c + K.dot(h_tm1 * B_U[2], self.U_c))
-        o = self.inner_activation(x_o + K.dot(h_tm1 * B_U[3], self.U_o))
+            z0 = z[:, :self.output_dim]
+            z1 = z[:, self.output_dim: 2 * self.output_dim]
+            z2 = z[:, 2 * self.output_dim: 3 * self.output_dim]
+            z3 = z[:, 3 * self.output_dim:]
+
+            i = self.inner_activation(z0)
+            f = self.inner_activation(z1)
+            c = f * c_tm1 + i * self.activation(z2)
+            o = self.inner_activation(z3)
+        else:
+            if self.consume_less == 'cpu':
+                x_i = x[:, :self.output_dim]
+                x_f = x[:, self.output_dim: 2 * self.output_dim]
+                x_c = x[:, 2 * self.output_dim: 3 * self.output_dim]
+                x_o = x[:, 3 * self.output_dim:]
+            elif self.consume_less == 'mem':
+                x_i = K.dot(x * B_W[0], self.W_i) + self.b_i
+                x_f = K.dot(x * B_W[1], self.W_f) + self.b_f
+                x_c = K.dot(x * B_W[2], self.W_c) + self.b_c
+                x_o = K.dot(x * B_W[3], self.W_o) + self.b_o
+            else:
+                raise Exception('Unknown `consume_less` mode.')
+
+            i = self.inner_activation(x_i + K.dot(h_tm1 * B_U[0], self.U_i))
+            f = self.inner_activation(x_f + K.dot(h_tm1 * B_U[1], self.U_f))
+            c = f * c_tm1 + i * self.activation(x_c + K.dot(h_tm1 * B_U[2], self.U_c))
+            o = self.inner_activation(x_o + K.dot(h_tm1 * B_U[3], self.U_o))
 
         h = o * self.activation(c)
         return h, [h, c]
@@ -784,16 +843,16 @@ class LSTM(Recurrent):
         return constants
 
     def get_config(self):
-        config = {"output_dim": self.output_dim,
-                  "init": self.init.__name__,
-                  "inner_init": self.inner_init.__name__,
-                  "forget_bias_init": self.forget_bias_init.__name__,
-                  "activation": self.activation.__name__,
-                  "inner_activation": self.inner_activation.__name__,
-                  "W_regularizer": self.W_regularizer.get_config() if self.W_regularizer else None,
-                  "U_regularizer": self.U_regularizer.get_config() if self.U_regularizer else None,
-                  "b_regularizer": self.b_regularizer.get_config() if self.b_regularizer else None,
-                  "dropout_W": self.dropout_W,
-                  "dropout_U": self.dropout_U}
+        config = {'output_dim': self.output_dim,
+                  'init': self.init.__name__,
+                  'inner_init': self.inner_init.__name__,
+                  'forget_bias_init': self.forget_bias_init.__name__,
+                  'activation': self.activation.__name__,
+                  'inner_activation': self.inner_activation.__name__,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'U_regularizer': self.U_regularizer.get_config() if self.U_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'dropout_W': self.dropout_W,
+                  'dropout_U': self.dropout_U}
         base_config = super(LSTM, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))

keras/layers/wrappers.py

@@ -6,6 +6,7 @@ class Wrapper(Layer):
 
     def __init__(self, layer, **kwargs):
         self.layer = layer
+        self.uses_learning_phase = layer.uses_learning_phase
         super(Wrapper, self).__init__(**kwargs)
 
     def build(self, input_shape=None):
@@ -97,7 +98,9 @@ class TimeDistributed(Wrapper):
                                 'an "input_shape" or "batch_input_shape" '
                                 'argument, including the time axis.')
         child_input_shape = (input_shape[0],) + input_shape[2:]
-        self.layer.build(child_input_shape)
+        if not self.layer.built:
+            self.layer.build(child_input_shape)
+            self.layer.built = True
         super(TimeDistributed, self).build()
 
     def get_output_shape_for(self, input_shape):
@@ -121,11 +124,11 @@ class TimeDistributed(Wrapper):
             # no batch size specified, therefore the layer will be able
             # to process batches of any size
             # we can go with reshape-based implementation for performance
-            X = K.reshape(X, (-1, ) + input_shape[2:])  # (nb_samples * timesteps, ...)
-            y = self.layer.call(X)  # (nb_samples * timesteps, ...)
             input_length = input_shape[1]
             if not input_length:
                 input_length = K.shape(X)[1]
+            X = K.reshape(X, (-1, ) + input_shape[2:])  # (nb_samples * timesteps, ...)
+            y = self.layer.call(X)  # (nb_samples * timesteps, ...)
             # (nb_samples, timesteps, ...)
             output_shape = self.get_output_shape_for(input_shape)
             y = K.reshape(y, (-1, input_length) + output_shape[2:])

keras/legacy/models.py

@@ -384,7 +384,7 @@ class Graph(Model):
 
         # Arguments
             data: dictionary mapping input names and outputs names to
-                appropriate numpy arrays. All arrays should contain
+                appropriate Numpy arrays. All arrays should contain
                 the same number of samples.
             batch_size: int. Number of samples per gradient update.
             nb_epoch: int.
@@ -395,7 +395,7 @@ class Graph(Model):
             validation_split: float (0. < x < 1). Fraction of the data to
                 use as held-out validation data.
             validation_data: dictionary mapping input names and outputs names
-                to appropriate numpy arrays to be used as
+                to appropriate Numpy arrays to be used as
                 held-out validation data.
                 All arrays should contain the same number of samples.
                 Will override validation_split.
@@ -473,6 +473,8 @@ class Graph(Model):
         x = self._get_x(data)
         output_list = super(Graph, self).predict(x, batch_size=batch_size,
                                                  verbose=verbose)
+        if not isinstance(output_list, list):
+            output_list = [output_list]
         return dict(zip(self._graph_outputs, output_list))
 
     def train_on_batch(self, data,
@@ -528,6 +530,8 @@ class Graph(Model):
 
     def predict_on_batch(self, data):
         output_list = super(Graph, self).predict_on_batch(data)
+        if not isinstance(output_list, list):
+            output_list = [output_list]
         return dict(zip(self._graph_outputs, output_list))
 
     def fit_generator(self, generator, samples_per_epoch, nb_epoch,
@@ -556,7 +560,7 @@ class Graph(Model):
             verbose: verbosity mode, 0, 1, or 2.
             callbacks: list of callbacks to be called during training.
             validation_data: dictionary mapping input names and outputs names
-                to appropriate numpy arrays to be used as
+                to appropriate Numpy arrays to be used as
                 held-out validation data, or a generator yielding such
                 dictionaries. All arrays should contain the same number
                 of samples. If a generator, will be called until more than
@@ -578,7 +582,7 @@ class Graph(Model):
                 while 1:
                     f = open(path)
                     for line in f:
-                        # create numpy arrays of input data
+                        # create Numpy arrays of input data
                         # and labels, from each line in the file
                         x1, x2, y = process_line(line)
                         yield ({'input_1': x1, 'input_2': x2, 'output': y})

keras/metrics.py

@@ -1,3 +1,4 @@
+import numpy as np
 from . import backend as K
 
 
@@ -10,6 +11,74 @@ def categorical_accuracy(y_true, y_pred):
                   K.argmax(y_pred, axis=-1)))
 
 
+def sparse_categorical_accuracy(y_true, y_pred):
+    return K.mean(K.equal(K.max(y_true, axis=-1),
+                          K.cast(K.argmax(y_pred, axis=-1), K.floatx())))
+
+
+def mean_squared_error(y_true, y_pred):
+    return K.mean(K.square(y_pred - y_true))
+
+
+def mean_absolute_error(y_true, y_pred):
+    return K.mean(K.abs(y_pred - y_true))
+
+
+def mean_absolute_percentage_error(y_true, y_pred):
+    diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), np.inf))
+    return 100. * K.mean(diff)
+
+
+def mean_squared_logarithmic_error(y_true, y_pred):
+    first_log = K.log(K.clip(y_pred, K.epsilon(), np.inf) + 1.)
+    second_log = K.log(K.clip(y_true, K.epsilon(), np.inf) + 1.)
+    return K.mean(K.square(first_log - second_log))
+
+
+def squared_hinge(y_true, y_pred):
+    return K.mean(K.square(K.maximum(1. - y_true * y_pred, 0.)))
+
+
+def hinge(y_true, y_pred):
+    return K.mean(K.maximum(1. - y_true * y_pred, 0.))
+
+
+def categorical_crossentropy(y_true, y_pred):
+    '''Expects a binary class matrix instead of a vector of scalar classes.
+    '''
+    return K.mean(K.categorical_crossentropy(y_pred, y_true))
+
+
+def sparse_categorical_crossentropy(y_true, y_pred):
+    '''expects an array of integer classes.
+    Note: labels shape must have the same number of dimensions as output shape.
+    If you get a shape error, add a length-1 dimension to labels.
+    '''
+    return K.mean(K.sparse_categorical_crossentropy(y_pred, y_true))
+
+
+def binary_crossentropy(y_true, y_pred):
+    return K.mean(K.binary_crossentropy(y_pred, y_true))
+
+
+def poisson(y_true, y_pred):
+    return K.mean(y_pred - y_true * K.log(y_pred + K.epsilon()))
+
+
+def cosine_proximity(y_true, y_pred):
+    y_true = K.l2_normalize(y_true, axis=-1)
+    y_pred = K.l2_normalize(y_pred, axis=-1)
+    return -K.mean(y_true * y_pred)
+
+
+# aliases
+mse = MSE = mean_squared_error
+mae = MAE = mean_absolute_error
+mape = MAPE = mean_absolute_percentage_error
+msle = MSLE = mean_squared_logarithmic_error
+cosine = cosine_proximity
+
+
 from .utils.generic_utils import get_from_module
 def get(identifier):
     return get_from_module(identifier, globals(), 'metric')

keras/models.py

@@ -8,11 +8,18 @@ from .engine.topology import get_source_inputs, Node
 from .legacy.models import Graph
 
 
+def model_from_config(config, custom_objects={}):
+    from keras.utils.layer_utils import layer_from_config
+    if isinstance(config, list):
+        raise Exception('`model_fom_config` expects a dictionary, not a list. '
+                        'Maybe you meant to use `Sequential.from_config(config)`?')
+    return layer_from_config(config, custom_objects=custom_objects)
+
+
 def model_from_yaml(yaml_string, custom_objects={}):
     '''Parses a yaml model configuration file
     and returns a model instance.
     '''
-    # TODO: legacy support?
     import yaml
     from keras.utils.layer_utils import layer_from_config
     config = yaml.load(yaml_string)
@@ -23,7 +30,6 @@ def model_from_json(json_string, custom_objects={}):
     '''Parses a JSON model configuration file
     and returns a model instance.
     '''
-    # TODO: legacy support?
     import json
     from keras.utils.layer_utils import layer_from_config
     config = json.loads(json_string)
@@ -76,6 +82,7 @@ class Sequential(Model):
         self.inbound_nodes = []
         self.outbound_nodes = []
         self.built = False
+        self._flattened_layers = None
 
         if not name:
             prefix = 'sequential_'
@@ -149,6 +156,7 @@ class Sequential(Model):
 
         self.layers.append(layer)
         self.built = False
+        self._flattened_layers = None
 
     def call(self, x, mask=None):
         if not self.built:
@@ -191,6 +199,8 @@ class Sequential(Model):
 
     @property
     def flattened_layers(self):
+        if self._flattened_layers is not None:
+            return self._flattened_layers
         layers = []
         if self.layers[0].__class__.__name__ == 'Merge':
             merge = self.layers[0]
@@ -212,6 +222,7 @@ class Sequential(Model):
         for layer in self.layers[1:]:
             if layer not in layers:
                 layers.append(layer)
+        self._flattened_layers = layers
         return layers
 
     def _gather_list_attr(self, attr):
@@ -449,12 +460,14 @@ class Sequential(Model):
             A Numpy array of predictions.
         '''
         if self.model is None:
-            raise Exception('The model needs to be compiled before being used.')
+            self.build()
         return self.model.predict(x, batch_size=batch_size, verbose=verbose)
 
     def predict_on_batch(self, x):
         '''Returns predictions for a single batch of samples.
         '''
+        if self.model is None:
+            self.build()
         return self.model.predict_on_batch(x)
 
     def train_on_batch(self, x, y, class_weight=None,
@@ -475,6 +488,8 @@ class Sequential(Model):
             The attribute `model.metrics_names` will give you
             the display labels for the scalar outputs.
         '''
+        if self.model is None:
+            raise Exception('The model needs to be compiled before being used.')
         if 'accuracy' in kwargs:
             kwargs.pop('accuracy')
             warnings.warn('The "accuracy" argument is deprecated, '
@@ -505,6 +520,8 @@ class Sequential(Model):
             The attribute `model.metrics_names` will give you
             the display labels for the scalar outputs.
         '''
+        if self.model is None:
+            raise Exception('The model needs to be compiled before being used.')
         if 'accuracy' in kwargs:
             kwargs.pop('accuracy')
             warnings.warn('The "accuracy" argument is deprecated, '
@@ -531,8 +548,6 @@ class Sequential(Model):
         # Returns
             A Numpy array of probability predictions.
         '''
-        if self.model is None:
-            raise Exception('The model needs to be compiled before being used.')
         preds = self.predict(x, batch_size, verbose)
         if preds.min() < 0. or preds.max() > 1.:
             warnings.warn('Network returning invalid probability values. '
@@ -554,8 +569,6 @@ class Sequential(Model):
         # Returns
             A numpy array of class predictions.
         '''
-        if self.model is None:
-            raise Exception('The model needs to be compiled before being used.')
         proba = self.predict(x, batch_size=batch_size, verbose=verbose)
         if proba.shape[-1] > 1:
             return proba.argmax(axis=-1)
@@ -607,7 +620,7 @@ class Sequential(Model):
                 while 1:
                     f = open(path)
                     for line in f:
-                        # create numpy arrays of input data
+                        # create Numpy arrays of input data
                         # and labels, from each line in the file
                         x, y = process_line(line)
                         yield (x, y)
@@ -694,13 +707,13 @@ class Sequential(Model):
             A Numpy array of predictions.
         '''
         if self.model is None:
-            raise Exception('The model needs to be compiled before being used.')
+            self.build()
         return self.model.predict_generator(generator, val_samples,
                                             max_q_size=max_q_size)
 
     def get_config(self):
         '''Returns the model configuration
-        as a Python dictionary.
+        as a Python list.
         '''
         config = []
         if self.layers[0].__class__.__name__ == 'Merge':
@@ -722,13 +735,16 @@ class Sequential(Model):
         return copy.deepcopy(config)
 
     @classmethod
-    def from_config(cls, config):
+    def from_config(cls, config, layer_cache=None):
         '''Supports legacy formats
         '''
         from keras.utils.layer_utils import layer_from_config
         from keras.layers import Merge
         assert type(config) is list
 
+        if not layer_cache:
+            layer_cache = {}
+
         def normalize_legacy_config(conf):
             if 'class_name' not in conf:
                 class_name = conf['name']
@@ -741,8 +757,20 @@ class Sequential(Model):
                 return new_config
             return conf
 
+        # the model we will return
         model = cls()
 
+        def get_or_create_layer(layer_data):
+            if layer_data['class_name'] == 'Sequential':
+                return Sequential.from_config(layer_data['config'],
+                                              layer_cache=layer_cache)
+            name = layer_data['config'].get('name')
+            if name in layer_cache:
+                return layer_cache[name]
+            layer = layer_from_config(layer_data)
+            layer_cache[name] = layer
+            return layer
+
         first_layer = config[0]
         first_layer = normalize_legacy_config(first_layer)
         if first_layer['class_name'] == 'Merge':
@@ -755,11 +783,11 @@ class Sequential(Model):
             merge = Merge.from_config(first_layer_config)
             model.add(merge)
         else:
-            layer = layer_from_config(first_layer)
+            layer = get_or_create_layer(first_layer)
             model.add(layer)
 
         for conf in config[1:]:
             conf = normalize_legacy_config(conf)
-            layer = layer_from_config(conf)
+            layer = get_or_create_layer(conf)
             model.add(layer)
         return model

keras/objectives.py

@@ -37,7 +37,9 @@ def categorical_crossentropy(y_true, y_pred):
 
 
 def sparse_categorical_crossentropy(y_true, y_pred):
-    '''expects a 1-D or 2-D array of integer classes.
+    '''expects an array of integer classes.
+    Note: labels shape must have the same number of dimensions as output shape.
+    If you get a shape error, add a length-1 dimension to labels.
     '''
     return K.sparse_categorical_crossentropy(y_pred, y_true)
 
@@ -46,6 +48,12 @@ def binary_crossentropy(y_true, y_pred):
     return K.mean(K.binary_crossentropy(y_pred, y_true), axis=-1)
 
 
+def kullback_leibler_divergence(y_true, y_pred):
+    y_true = K.clip(y_true, K.epsilon(), 1)
+    y_pred = K.clip(y_pred, K.epsilon(), 1)
+    return K.sum(y_true * K.log(y_true / y_pred), axis=-1)
+
+
 def poisson(y_true, y_pred):
     return K.mean(y_pred - y_true * K.log(y_pred + K.epsilon()), axis=-1)
 
@@ -61,6 +69,7 @@ mse = MSE = mean_squared_error
 mae = MAE = mean_absolute_error
 mape = MAPE = mean_absolute_percentage_error
 msle = MSLE = mean_squared_logarithmic_error
+kld = KLD = kullback_leibler_divergence
 cosine = cosine_proximity
 
 from .utils.generic_utils import get_from_module

keras/optimizers.py

@@ -29,6 +29,11 @@ class Optimizer(object):
             when their absolute value exceeds this value.
     '''
     def __init__(self, **kwargs):
+        allowed_kwargs = {'clipnorm', 'clipvalue'}
+        for k in kwargs:
+            if k not in allowed_kwargs:
+                raise Exception('Unexpected keyword argument '
+                                'passed to optimizer: ' + str(k))
         self.__dict__.update(kwargs)
         self.updates = []
         self.weights = []
@@ -89,7 +94,12 @@ class Optimizer(object):
         return weights
 
     def get_config(self):
-        return {"name": self.__class__.__name__}
+        config = {'name': self.__class__.__name__}
+        if hasattr(self, 'clipnorm'):
+            config['clipnorm'] = self.clipnorm
+        if hasattr(self, 'clipvalue'):
+            config['clipvalue'] = self.clipvalue
+        return config
 
 
 class SGD(Optimizer):
@@ -102,8 +112,8 @@ class SGD(Optimizer):
         decay: float >= 0. Learning rate decay over each update.
         nesterov: boolean. Whether to apply Nesterov momentum.
     '''
-    def __init__(self, lr=0.01, momentum=0., decay=0., nesterov=False,
-                 *args, **kwargs):
+    def __init__(self, lr=0.01, momentum=0., decay=0.,
+                 nesterov=False, **kwargs):
         super(SGD, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.iterations = K.variable(0.)
@@ -116,8 +126,9 @@ class SGD(Optimizer):
         lr = self.lr * (1. / (1. + self.decay * self.iterations))
         self.updates = [(self.iterations, self.iterations + 1.)]
 
-        for p, g in zip(params, grads):
-            m = K.variable(np.zeros(K.get_value(p).shape))  # momentum
+        # momentum
+        self.weights = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
+        for p, g, m in zip(params, grads, self.weights):
             v = self.momentum * m - lr * g  # velocity
             self.updates.append((m, v))
 
@@ -134,11 +145,12 @@ class SGD(Optimizer):
         return self.updates
 
     def get_config(self):
-        return {"name": self.__class__.__name__,
-                "lr": float(K.get_value(self.lr)),
-                "momentum": float(K.get_value(self.momentum)),
-                "decay": float(K.get_value(self.decay)),
-                "nesterov": self.nesterov}
+        config = {'lr': float(K.get_value(self.lr)),
+                  'momentum': float(K.get_value(self.momentum)),
+                  'decay': float(K.get_value(self.decay)),
+                  'nesterov': self.nesterov}
+        base_config = super(SGD, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
 
 
 class RMSprop(Optimizer):
@@ -156,7 +168,7 @@ class RMSprop(Optimizer):
         rho: float >= 0.
         epsilon: float >= 0. Fuzz factor.
     '''
-    def __init__(self, lr=0.001, rho=0.9, epsilon=1e-6, *args, **kwargs):
+    def __init__(self, lr=0.001, rho=0.9, epsilon=1e-8, **kwargs):
         super(RMSprop, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.lr = K.variable(lr)
@@ -172,7 +184,7 @@ class RMSprop(Optimizer):
             # update accumulator
             new_a = self.rho * a + (1. - self.rho) * K.square(g)
             self.updates.append((a, new_a))
-            new_p = p - self.lr * g / K.sqrt(new_a + self.epsilon)
+            new_p = p - self.lr * g / (K.sqrt(new_a) + self.epsilon)
 
             # apply constraints
             if p in constraints:
@@ -182,10 +194,11 @@ class RMSprop(Optimizer):
         return self.updates
 
     def get_config(self):
-        return {"name": self.__class__.__name__,
-                "lr": float(K.get_value(self.lr)),
-                "rho": float(K.get_value(self.rho)),
-                "epsilon": self.epsilon}
+        config = {'lr': float(K.get_value(self.lr)),
+                  'rho': float(K.get_value(self.rho)),
+                  'epsilon': self.epsilon}
+        base_config = super(RMSprop, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
 
 
 class Adagrad(Optimizer):
@@ -198,7 +211,7 @@ class Adagrad(Optimizer):
         lr: float >= 0. Learning rate.
         epsilon: float >= 0.
     '''
-    def __init__(self, lr=0.01, epsilon=1e-6, *args, **kwargs):
+    def __init__(self, lr=0.01, epsilon=1e-8, **kwargs):
         super(Adagrad, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.lr = K.variable(lr)
@@ -212,7 +225,7 @@ class Adagrad(Optimizer):
         for p, g, a in zip(params, grads, self.weights):
             new_a = a + K.square(g)  # update accumulator
             self.updates.append((a, new_a))
-            new_p = p - self.lr * g / K.sqrt(new_a + self.epsilon)
+            new_p = p - self.lr * g / (K.sqrt(new_a) + self.epsilon)
             # apply constraints
             if p in constraints:
                 c = constraints[p]
@@ -221,9 +234,10 @@ class Adagrad(Optimizer):
         return self.updates
 
     def get_config(self):
-        return {"name": self.__class__.__name__,
-                "lr": float(K.get_value(self.lr)),
-                "epsilon": self.epsilon}
+        config = {'lr': float(K.get_value(self.lr)),
+                  'epsilon': self.epsilon}
+        base_config = super(Adagrad, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
 
 
 class Adadelta(Optimizer):
@@ -241,7 +255,7 @@ class Adadelta(Optimizer):
     # References
         - [Adadelta - an adaptive learning rate method](http://arxiv.org/abs/1212.5701)
     '''
-    def __init__(self, lr=1.0, rho=0.95, epsilon=1e-6, *args, **kwargs):
+    def __init__(self, lr=1.0, rho=0.95, epsilon=1e-8, **kwargs):
         super(Adadelta, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.lr = K.variable(lr)
@@ -274,10 +288,11 @@ class Adadelta(Optimizer):
         return self.updates
 
     def get_config(self):
-        return {"name": self.__class__.__name__,
-                "lr": float(K.get_value(self.lr)),
-                "rho": self.rho,
-                "epsilon": self.epsilon}
+        config = {'lr': float(K.get_value(self.lr)),
+                  'rho': self.rho,
+                  'epsilon': self.epsilon}
+        base_config = super(Adadelta, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
 
 
 class Adam(Optimizer):
@@ -293,8 +308,8 @@ class Adam(Optimizer):
     # References
         - [Adam - A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980v8)
     '''
-    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8,
-                 *args, **kwargs):
+    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
+                 epsilon=1e-8, **kwargs):
         super(Adam, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.iterations = K.variable(0)
@@ -330,11 +345,12 @@ class Adam(Optimizer):
         return self.updates
 
     def get_config(self):
-        return {"name": self.__class__.__name__,
-                "lr": float(K.get_value(self.lr)),
-                "beta_1": float(K.get_value(self.beta_1)),
-                "beta_2": float(K.get_value(self.beta_2)),
-                "epsilon": self.epsilon}
+        config = {'lr': float(K.get_value(self.lr)),
+                  'beta_1': float(K.get_value(self.beta_1)),
+                  'beta_2': float(K.get_value(self.beta_2)),
+                  'epsilon': self.epsilon}
+        base_config = super(Adam, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
 
 
 class Adamax(Optimizer):
@@ -351,8 +367,8 @@ class Adamax(Optimizer):
     # References
         - [Adam - A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980v8)
     '''
-    def __init__(self, lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-8,
-                 *args, **kwargs):
+    def __init__(self, lr=0.002, beta_1=0.9, beta_2=0.999,
+                 epsilon=1e-8, **kwargs):
         super(Adamax, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.iterations = K.variable(0.)
@@ -391,11 +407,92 @@ class Adamax(Optimizer):
         return self.updates
 
     def get_config(self):
-        return {"name": self.__class__.__name__,
-                "lr": float(K.get_value(self.lr)),
-                "beta_1": float(K.get_value(self.beta_1)),
-                "beta_2": float(K.get_value(self.beta_2)),
-                "epsilon": self.epsilon}
+        config = {'lr': float(K.get_value(self.lr)),
+                  'beta_1': float(K.get_value(self.beta_1)),
+                  'beta_2': float(K.get_value(self.beta_2)),
+                  'epsilon': self.epsilon}
+        base_config = super(Adamax, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class Nadam(Optimizer):
+    '''
+    Nesterov Adam optimizer: Much like Adam is essentially RMSprop with momentum,
+    Nadam is Adam RMSprop with Nesterov momentum.
+
+    Default parameters follow those provided in the paper.
+    It is recommended to leave the parameters of this optimizer
+    at their default values.
+
+    # Arguments
+        lr: float >= 0. Learning rate.
+        beta_1/beta_2: floats, 0 < beta < 1. Generally close to 1.
+        epsilon: float >= 0. Fuzz factor.
+
+    # References
+        [1] Nadam report - http://cs229.stanford.edu/proj2015/054_report.pdf
+        [2] On the importance of initialization and momentum in deep learning -
+            http://www.cs.toronto.edu/~fritz/absps/momentum.pdf
+    '''
+    def __init__(self, lr=0.002, beta_1=0.9, beta_2=0.999,
+                 epsilon=1e-8, schedule_decay=0.004, **kwargs):
+        super(Nadam, self).__init__(**kwargs)
+        self.__dict__.update(locals())
+        self.iterations = K.variable(0.)
+        self.m_schedule = K.variable(1.)
+        self.lr = K.variable(lr)
+        self.beta_1 = K.variable(beta_1)
+        self.beta_2 = K.variable(beta_2)
+        self.schedule_decay = schedule_decay
+
+    def get_updates(self, params, constraints, loss):
+        grads = self.get_gradients(loss, params)
+        self.updates = [(self.iterations, self.iterations + 1)]
+
+        t = self.iterations + 1
+
+        # Due to the recommendations in [2], i.e. warming momentum schedule
+        momentum_cache_t = self.beta_1 * (1. - 0.5 * (K.pow(0.96, t * self.schedule_decay)))
+        momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (K.pow(0.96, (t + 1) * self.schedule_decay)))
+        m_schedule_new = self.m_schedule * momentum_cache_t
+        m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
+        self.updates.append((self.m_schedule, m_schedule_new))
+
+        ms = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
+        vs = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
+
+        self.weights = ms + vs
+
+        for p, g, m, v in zip(params, grads, ms, vs):
+            # the following equations given in [1]
+            g_prime = g / (1. - m_schedule_new)
+            m_t = self.beta_1 * m + (1. - self.beta_1) * g
+            m_t_prime = m_t / (1. - m_schedule_next)
+            v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g)
+            v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
+            m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime
+
+            self.updates.append((m, m_t))
+            self.updates.append((v, v_t))
+
+            p_t = p - self.lr * m_t_bar / (K.sqrt(v_t_prime) + self.epsilon)
+            new_p = p_t
+
+            # apply constraints
+            if p in constraints:
+                c = constraints[p]
+                new_p = c(new_p)
+            self.updates.append((p, new_p))
+        return self.updates
+
+    def get_config(self):
+        config = {'lr': float(K.get_value(self.lr)),
+                  'beta_1': float(K.get_value(self.beta_1)),
+                  'beta_2': float(K.get_value(self.beta_2)),
+                  'epsilon': self.epsilon,
+                  'schedule_decay': self.schedule_decay}
+        base_config = super(Nadam, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
 
 
 # aliases
@@ -405,6 +502,7 @@ adagrad = Adagrad
 adadelta = Adadelta
 adam = Adam
 adamax = Adamax
+nadam = Nadam
 
 
 def get(identifier, kwargs=None):

keras/preprocessing/image.py

@@ -1,43 +1,113 @@
-'''Fairly basic set of tools for realtime data augmentation on image data.
+'''Fairly basic set of tools for real-time data augmentation on image data.
 Can easily be extended to include new transformations,
 new preprocessing methods, etc...
 '''
 from __future__ import absolute_import
+from __future__ import print_function
 
 import numpy as np
 import re
-from scipy import ndimage
 from scipy import linalg
-
-from os import listdir
-from os.path import isfile, join
-import math
+import scipy.ndimage as ndi
 from six.moves import range
+import os
 import threading
 
+from .. import backend as K
 
-def random_rotation(x, rg, fill_mode='nearest',
-                    cval=0., axes=(1, 2)):
-    angle = np.random.uniform(-rg, rg)
-    x = ndimage.interpolation.rotate(x, angle,
-                                     axes=axes,
-                                     reshape=False,
-                                     mode=fill_mode,
-                                     cval=cval)
+
+def random_rotation(x, rg, row_index=1, col_index=2, channel_index=0,
+                    fill_mode='nearest', cval=0.):
+    theta = np.pi / 180 * np.random.uniform(-rg, rg)
+    rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0],
+                                [np.sin(theta), np.cos(theta), 0],
+                                [0, 0, 1]])
+
+    h, w = x.shape[row_index], x.shape[col_index]
+    transform_matrix = transform_matrix_offset_center(rotation_matrix, h, w)
+    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
     return x
 
 
-def random_shift(x, wrg, hrg, fill_mode='nearest',
-                 cval=0., row_index=1, col_index=2):
-    shift_x = shift_y = 0
-    if wrg:
-        shift_x = np.random.uniform(-wrg, wrg) * x.shape[col_index]
-    if hrg:
-        shift_y = np.random.uniform(-hrg, hrg) * x.shape[row_index]
-    x = ndimage.interpolation.shift(x, (0, shift_y, shift_x),
-                                    order=0,
-                                    mode=fill_mode,
-                                    cval=cval)
+def random_shift(x, wrg, hrg, row_index=1, col_index=2, channel_index=0,
+                 fill_mode='nearest', cval=0.):
+    h, w = x.shape[row_index], x.shape[col_index]
+    tx = np.random.uniform(-hrg, hrg) * h
+    ty = np.random.uniform(-wrg, wrg) * w
+    translation_matrix = np.array([[1, 0, tx],
+                                   [0, 1, ty],
+                                   [0, 0, 1]])
+
+    transform_matrix = translation_matrix  # no need to do offset
+    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
+    return x
+
+
+def random_shear(x, intensity, row_index=1, col_index=2, channel_index=0,
+                 fill_mode='nearest', cval=0.):
+    shear = np.random.uniform(-intensity, intensity)
+    shear_matrix = np.array([[1, -np.sin(shear), 0],
+                             [0, np.cos(shear), 0],
+                             [0, 0, 1]])
+
+    h, w = x.shape[row_index], x.shape[col_index]
+    transform_matrix = transform_matrix_offset_center(shear_matrix, h, w)
+    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
+    return x
+
+
+def random_zoom(x, zoom_range, row_index=1, col_index=2, channel_index=0,
+                fill_mode='nearest', cval=0.):
+    if len(zoom_range) != 2:
+        raise Exception('zoom_range should be a tuple or list of two floats. '
+                        'Received arg: ', zoom_range)
+
+    if zoom_range[0] == 1 and zoom_range[1] == 1:
+        zx, zy = 1, 1
+    else:
+        zx, zy = np.random.uniform(zoom_range[0], zoom_range[1], 2)
+    zoom_matrix = np.array([[zx, 0, 0],
+                            [0, zy, 0],
+                            [0, 0, 1]])
+
+    h, w = x.shape[row_index], x.shape[col_index]
+    transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w)
+    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
+    return x
+
+
+def random_barrel_transform(x, intensity):
+    # TODO
+    pass
+
+
+def random_channel_shift(x, intensity, channel_index=0):
+    x = np.rollaxis(x, channel_index, 0)
+    min_x, max_x = np.min(x), np.max(x)
+    channel_images = [np.clip(x_channel + np.random.uniform(-intensity, intensity), min_x, max_x)
+                      for x_channel in x]
+    x = np.stack(channel_images, axis=0)
+    x = np.rollaxis(x, 0, channel_index+1)
+    return x
+
+
+def transform_matrix_offset_center(matrix, x, y):
+    o_x = float(x) / 2 + 0.5
+    o_y = float(y) / 2 + 0.5
+    offset_matrix = np.array([[1, 0, o_x], [0, 1, o_y], [0, 0, 1]])
+    reset_matrix = np.array([[1, 0, -o_x], [0, 1, -o_y], [0, 0, 1]])
+    transform_matrix = np.dot(np.dot(offset_matrix, matrix), reset_matrix)
+    return transform_matrix
+
+
+def apply_transform(x, transform_matrix, channel_index=0, fill_mode='nearest', cval=0.):
+    x = np.rollaxis(x, channel_index, 0)
+    final_affine_matrix = transform_matrix[:2, :2]
+    final_offset = transform_matrix[:2, 2]
+    channel_images = [ndi.interpolation.affine_transform(x_channel, final_affine_matrix,
+                      final_offset, order=0, mode=fill_mode, cval=cval) for x_channel in x]
+    x = np.stack(channel_images, axis=0)
+    x = np.rollaxis(x, 0, channel_index+1)
     return x
 
 
@@ -48,40 +118,10 @@ def flip_axis(x, axis):
     return x
 
 
-def random_barrel_transform(x, intensity):
-    # TODO
-    pass
-
-
-def random_shear(x, intensity, fill_mode='nearest', cval=0.):
-    shear = np.random.uniform(-intensity, intensity)
-    shear_matrix = np.array([[1.0, -math.sin(shear), 0.0],
-                            [0.0, math.cos(shear), 0.0],
-                            [0.0, 0.0, 1.0]])
-    x = ndimage.interpolation.affine_transform(x, shear_matrix,
-                                               mode=fill_mode,
-                                               order=3,
-                                               cval=cval)
-    return x
-
-
-def random_channel_shift(x, rg):
-    # TODO
-    pass
-
-
-def random_zoom(x, rg, fill_mode='nearest', cval=0.):
-    zoom_w = np.random.uniform(1.-rg, 1.)
-    zoom_h = np.random.uniform(1.-rg, 1.)
-    x = ndimage.interpolation.zoom(x, zoom=(1., zoom_w, zoom_h),
-                                   mode=fill_mode,
-                                   cval=cval)
-    return x  # shape of result will be different from shape of input!
-
-
-def array_to_img(x, scale=True):
+def array_to_img(x, dim_ordering=K.image_dim_ordering(), scale=True):
     from PIL import Image
-    x = x.transpose(1, 2, 0)
+    if dim_ordering == 'th':
+        x = x.transpose(1, 2, 0)
     if scale:
         x += max(-np.min(x), 0)
         x /= np.max(x)
@@ -89,35 +129,46 @@ def array_to_img(x, scale=True):
     if x.shape[2] == 3:
         # RGB
         return Image.fromarray(x.astype('uint8'), 'RGB')
-    else:
+    elif x.shape[2] == 1:
         # grayscale
         return Image.fromarray(x[:, :, 0].astype('uint8'), 'L')
+    else:
+        raise Exception('Unsupported channel number: ', x.shape[2])
 
 
-def img_to_array(img):
+def img_to_array(img, dim_ordering=K.image_dim_ordering()):
+    if dim_ordering not in ['th', 'tf']:
+        raise Exception('Unknown dim_ordering: ', dim_ordering)
+    # image has dim_ordering (height, width, channel)
     x = np.asarray(img, dtype='float32')
     if len(x.shape) == 3:
-        # RGB: height, width, channel -> channel, height, width
-        x = x.transpose(2, 0, 1)
+        if dim_ordering == 'th':
+            x = x.transpose(2, 0, 1)
+    elif len(x.shape) == 2:
+        if dim_ordering == 'th':
+            x = x.reshape((1, x.shape[0], x.shape[1]))
+        else:
+            x = x.reshape((x.shape[0], x.shape[1], 1))
     else:
-        # grayscale: height, width -> channel, height, width
-        x = x.reshape((1, x.shape[0], x.shape[1]))
+        raise Exception('Unsupported image shape: ', x.shape)
     return x
 
 
-def load_img(path, grayscale=False):
+def load_img(path, grayscale=False, target_size=None):
     from PIL import Image
     img = Image.open(path)
     if grayscale:
         img = img.convert('L')
     else:  # Ensure 3 channel even when loaded image is grayscale
         img = img.convert('RGB')
+    if target_size:
+        img = img.resize(target_size)
     return img
 
 
 def list_pictures(directory, ext='jpg|jpeg|bmp|png'):
-    return [join(directory, f) for f in listdir(directory)
-            if isfile(join(directory, f)) and re.match('([\w]+\.(?:' + ext + '))', f)]
+    return [os.path.join(directory, f) for f in os.listdir(directory)
+            if os.path.isfile(os.path.join(directory, f)) and re.match('([\w]+\.(?:' + ext + '))', f)]
 
 
 class ImageDataGenerator(object):
@@ -134,116 +185,103 @@ class ImageDataGenerator(object):
         width_shift_range: fraction of total width.
         height_shift_range: fraction of total height.
         shear_range: shear intensity (shear angle in radians).
+        zoom_range: amount of zoom. if scalar z, zoom will be randomly picked
+            in the range [1-z, 1+z]. A sequence of two can be passed instead
+            to select this range.
+        channel_shift_range: shift range for each channels.
+        fill_mode: points outside the boundaries are filled according to the
+            given mode ('constant', 'nearest', 'reflect' or 'wrap'). Default
+            is 'nearest'.
+        cval: value used for points outside the boundaries when fill_mode is
+            'constant'. Default is 0.
         horizontal_flip: whether to randomly flip images horizontally.
         vertical_flip: whether to randomly flip images vertically.
+        rescale: rescaling factor. If None or 0, no rescaling is applied,
+            otherwise we multiply the data by the value provided (before applying
+            any other transformation).
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode it is at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
     '''
     def __init__(self,
-                 featurewise_center=True,
+                 featurewise_center=False,
                  samplewise_center=False,
-                 featurewise_std_normalization=True,
+                 featurewise_std_normalization=False,
                  samplewise_std_normalization=False,
                  zca_whitening=False,
                  rotation_range=0.,
                  width_shift_range=0.,
                  height_shift_range=0.,
                  shear_range=0.,
+                 zoom_range=0.,
+                 channel_shift_range=0.,
+                 fill_mode='nearest',
+                 cval=0.,
                  horizontal_flip=False,
                  vertical_flip=False,
-                 dim_ordering='th'):
+                 rescale=None,
+                 dim_ordering=K.image_dim_ordering()):
         self.__dict__.update(locals())
         self.mean = None
         self.std = None
         self.principal_components = None
-        self.lock = threading.Lock()
+        self.rescale = rescale
+
         if dim_ordering not in {'tf', 'th'}:
-            raise Exception('dim_ordering should be "tf" (channel after row and \
-            column) or "th" (channel before row and column). Received arg: ', dim_ordering)
+            raise Exception('dim_ordering should be "tf" (channel after row and '
+                            'column) or "th" (channel before row and column). '
+                            'Received arg: ', dim_ordering)
         self.dim_ordering = dim_ordering
-        if dim_ordering == "th":
+        if dim_ordering == 'th':
             self.channel_index = 1
             self.row_index = 2
             self.col_index = 3
-        if dim_ordering == "tf":
+        if dim_ordering == 'tf':
             self.channel_index = 3
             self.row_index = 1
             self.col_index = 2
 
-    def _flow_index(self, N, batch_size=32, shuffle=False, seed=None):
-        b = 0
-        total_b = 0
-        while 1:
-            if b == 0:
-                if seed is not None:
-                    np.random.seed(seed + total_b)
+        if np.isscalar(zoom_range):
+            self.zoom_range = [1 - zoom_range, 1 + zoom_range]
+        elif len(zoom_range) == 2:
+            self.zoom_range = [zoom_range[0], zoom_range[1]]
+        else:
+            raise Exception('zoom_range should be a float or '
+                            'a tuple or list of two floats. '
+                            'Received arg: ', zoom_range)
 
-                if shuffle:
-                    index_array = np.random.permutation(N)
-                else:
-                    index_array = np.arange(N)
-
-            current_index = (b * batch_size) % N
-            if N >= current_index + batch_size:
-                current_batch_size = batch_size
-            else:
-                current_batch_size = N - current_index
-
-            if current_batch_size == batch_size:
-                b += 1
-            else:
-                b = 0
-            total_b += 1
-            yield (index_array[current_index: current_index + current_batch_size],
-                   current_index, current_batch_size)
-
-    def flow(self, X, y, batch_size=32, shuffle=False, seed=None,
+    def flow(self, X, y=None, batch_size=32, shuffle=True, seed=None,
              save_to_dir=None, save_prefix='', save_format='jpeg'):
-        assert len(X) == len(y)
-        self.X = X
-        self.y = y
-        self.save_to_dir = save_to_dir
-        self.save_prefix = save_prefix
-        self.save_format = save_format
-        self.flow_generator = self._flow_index(X.shape[0], batch_size,
-                                               shuffle, seed)
-        return self
+        return NumpyArrayIterator(
+            X, y, self,
+            batch_size=batch_size, shuffle=shuffle, seed=seed,
+            dim_ordering=self.dim_ordering,
+            save_to_dir=save_to_dir, save_prefix=save_prefix, save_format=save_format)
 
-    def __iter__(self):
-        # needed if we want to do something like:
-        # for x, y in data_gen.flow(...):
-        return self
-
-    def next(self):
-        # for python 2.x.
-        # Keeps under lock only the mechanism which advances
-        # the indexing of each batch
-        # see # http://anandology.com/blog/using-iterators-and-generators/
-        with self.lock:
-            index_array, current_index, current_batch_size = next(self.flow_generator)
-        # The transformation of images is not under thread lock so it can be done in parallel
-        bX = np.zeros(tuple([current_batch_size] + list(self.X.shape)[1:]))
-        for i, j in enumerate(index_array):
-            x = self.X[j]
-            x = self.random_transform(x.astype('float32'))
-            x = self.standardize(x)
-            bX[i] = x
-        if self.save_to_dir:
-            for i in range(current_batch_size):
-                img = array_to_img(bX[i], scale=True)
-                img.save(self.save_to_dir + '/' + self.save_prefix + '_' + str(current_index + i) + '.' + self.save_format)
-        bY = self.y[index_array]
-        return bX, bY
-
-    def __next__(self):
-        # for python 3.x.
-        return self.next()
+    def flow_from_directory(self, directory,
+                            target_size=(256, 256), color_mode='rgb',
+                            classes=None, class_mode='categorical',
+                            batch_size=32, shuffle=True, seed=None,
+                            save_to_dir=None, save_prefix='', save_format='jpeg'):
+        return DirectoryIterator(
+            directory, self,
+            target_size=target_size, color_mode=color_mode,
+            classes=classes, class_mode=class_mode,
+            dim_ordering=self.dim_ordering,
+            batch_size=batch_size, shuffle=shuffle, seed=seed,
+            save_to_dir=save_to_dir, save_prefix=save_prefix, save_format=save_format)
 
     def standardize(self, x):
+        if self.rescale:
+            x *= self.rescale
+        # x is a single image, so it doesn't have image number at index 0
+        img_channel_index = self.channel_index - 1
         if self.samplewise_center:
-            x -= np.mean(x, axis=self.channel_index, keepdims=True)
+            x -= np.mean(x, axis=img_channel_index, keepdims=True)
         if self.samplewise_std_normalization:
-            x /= (np.std(x, axis=self.channel_index, keepdims=True) + 1e-7)
+            x /= (np.std(x, axis=img_channel_index, keepdims=True) + 1e-7)
 
         if self.featurewise_center:
             x -= self.mean
@@ -259,27 +297,67 @@ class ImageDataGenerator(object):
 
     def random_transform(self, x):
         # x is a single image, so it doesn't have image number at index 0
-        img_col_index = self.col_index - 1
         img_row_index = self.row_index - 1
+        img_col_index = self.col_index - 1
+        img_channel_index = self.channel_index - 1
 
+        # use composition of homographies to generate final transform that needs to be applied
         if self.rotation_range:
-            x = random_rotation(x, self.rotation_range,
-                                axes=(img_row_index, img_col_index))
-        if self.width_shift_range or self.height_shift_range:
-            x = random_shift(x, self.width_shift_range, self.height_shift_range,
-                             row_index=img_row_index, col_index=img_col_index)
+            theta = np.pi / 180 * np.random.uniform(-self.rotation_range, self.rotation_range)
+        else:
+            theta = 0
+        rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0],
+                                    [np.sin(theta), np.cos(theta), 0],
+                                    [0, 0, 1]])
+        if self.height_shift_range:
+            tx = np.random.uniform(-self.height_shift_range, self.height_shift_range) * x.shape[img_row_index]
+        else:
+            tx = 0
+
+        if self.width_shift_range:
+            ty = np.random.uniform(-self.width_shift_range, self.width_shift_range) * x.shape[img_col_index]
+        else:
+            ty = 0
+
+        translation_matrix = np.array([[1, 0, tx],
+                                       [0, 1, ty],
+                                       [0, 0, 1]])
+        if self.shear_range:
+            shear = np.random.uniform(-self.shear_range, self.shear_range)
+        else:
+            shear = 0
+        shear_matrix = np.array([[1, -np.sin(shear), 0],
+                                 [0, np.cos(shear), 0],
+                                 [0, 0, 1]])
+
+        if self.zoom_range[0] == 1 and self.zoom_range[1] == 1:
+            zx, zy = 1, 1
+        else:
+            zx, zy = np.random.uniform(self.zoom_range[0], self.zoom_range[1], 2)
+        zoom_matrix = np.array([[zx, 0, 0],
+                                [0, zy, 0],
+                                [0, 0, 1]])
+
+        transform_matrix = np.dot(np.dot(np.dot(rotation_matrix, translation_matrix), shear_matrix), zoom_matrix)
+
+        h, w = x.shape[img_row_index], x.shape[img_col_index]
+        transform_matrix = transform_matrix_offset_center(transform_matrix, h, w)
+        x = apply_transform(x, transform_matrix, img_channel_index,
+                            fill_mode=self.fill_mode, cval=self.cval)
+        if self.channel_shift_range != 0:
+            x = random_channel_shift(x, self.channel_shift_range, img_channel_index)
+
         if self.horizontal_flip:
             if np.random.random() < 0.5:
                 x = flip_axis(x, img_col_index)
+
         if self.vertical_flip:
             if np.random.random() < 0.5:
                 x = flip_axis(x, img_row_index)
-        if self.shear_range:
-            x = random_shear(x, self.shear_range)
+
         # TODO:
-        # zoom
+        # channel-wise normalization
         # barrel/fisheye
-        # channel shifting
         return x
 
     def fit(self, X,
@@ -301,14 +379,13 @@ class ImageDataGenerator(object):
             aX = np.zeros(tuple([rounds * X.shape[0]] + list(X.shape)[1:]))
             for r in range(rounds):
                 for i in range(X.shape[0]):
-                    img = array_to_img(X[i])
-                    img = self.random_transform(img)
-                    aX[i + r * X.shape[0]] = img_to_array(img)
+                    aX[i + r * X.shape[0]] = self.random_transform(X[i])
             X = aX
 
         if self.featurewise_center:
             self.mean = np.mean(X, axis=0)
             X -= self.mean
+
         if self.featurewise_std_normalization:
             self.std = np.std(X, axis=0)
             X /= (self.std + 1e-7)
@@ -320,10 +397,209 @@ class ImageDataGenerator(object):
             self.principal_components = np.dot(np.dot(U, np.diag(1. / np.sqrt(S + 10e-7))), U.T)
 
 
-class GraphImageDataGenerator(ImageDataGenerator):
-    '''Example of how to build a generator for a Graph model
-    '''
+class Iterator(object):
+
+    def __init__(self, N, batch_size, shuffle, seed):
+        self.N = N
+        self.batch_size = batch_size
+        self.shuffle = shuffle
+        self.batch_index = 0
+        self.total_batches_seen = 0
+        self.lock = threading.Lock()
+        self.index_generator = self._flow_index(N, batch_size, shuffle, seed)
+
+    def reset(self):
+        self.batch_index = 0
+
+    def _flow_index(self, N, batch_size=32, shuffle=False, seed=None):
+        # ensure self.batch_index is 0
+        self.reset()
+        while 1:
+            if self.batch_index == 0:
+                index_array = np.arange(N)
+                if shuffle:
+                    if seed is not None:
+                        np.random.seed(seed + self.total_batches_seen)
+                    index_array = np.random.permutation(N)
+
+            current_index = (self.batch_index * batch_size) % N
+            if N >= current_index + batch_size:
+                current_batch_size = batch_size
+                self.batch_index += 1
+            else:
+                current_batch_size = N - current_index
+                self.batch_index = 0
+            self.total_batches_seen += 1
+            yield (index_array[current_index: current_index + current_batch_size],
+                   current_index, current_batch_size)
+
+    def __iter__(self):
+        # needed if we want to do something like:
+        # for x, y in data_gen.flow(...):
+        return self
+
+    def __next__(self, *args, **kwargs):
+        return self.next(*args, **kwargs)
+
+
+class NumpyArrayIterator(Iterator):
+
+    def __init__(self, X, y, image_data_generator,
+                 batch_size=32, shuffle=False, seed=None,
+                 dim_ordering=K.image_dim_ordering(),
+                 save_to_dir=None, save_prefix='', save_format='jpeg'):
+        if y is not None and len(X) != len(y):
+            raise Exception('X (images tensor) and y (labels) '
+                            'should have the same length. '
+                            'Found: X.shape = %s, y.shape = %s' % (np.asarray(X).shape, np.asarray(y).shape))
+        self.X = X
+        self.y = y
+        self.image_data_generator = image_data_generator
+        self.dim_ordering = dim_ordering
+        self.save_to_dir = save_to_dir
+        self.save_prefix = save_prefix
+        self.save_format = save_format
+        super(NumpyArrayIterator, self).__init__(X.shape[0], batch_size, shuffle, seed)
 
     def next(self):
-        bX, bY = super(GraphImageDataGenerator, self).next()
-        return {'input': bX, 'output': bY}
+        # for python 2.x.
+        # Keeps under lock only the mechanism which advances
+        # the indexing of each batch
+        # see http://anandology.com/blog/using-iterators-and-generators/
+        with self.lock:
+            index_array, current_index, current_batch_size = next(self.index_generator)
+        # The transformation of images is not under thread lock so it can be done in parallel
+        batch_x = np.zeros(tuple([current_batch_size] + list(self.X.shape)[1:]))
+        for i, j in enumerate(index_array):
+            x = self.X[j]
+            x = self.image_data_generator.random_transform(x.astype('float32'))
+            x = self.image_data_generator.standardize(x)
+            batch_x[i] = x
+        if self.save_to_dir:
+            for i in range(current_batch_size):
+                img = array_to_img(batch_x[i], self.dim_ordering, scale=True)
+                fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
+                                                                  index=current_index + i,
+                                                                  hash=np.random.randint(1e4),
+                                                                  format=self.save_format)
+                img.save(os.path.join(self.save_to_dir, fname))
+        if self.y is None:
+            return batch_x
+        batch_y = self.y[index_array]
+        return batch_x, batch_y
+
+
+class DirectoryIterator(Iterator):
+
+    def __init__(self, directory, image_data_generator,
+                 target_size=(256, 256), color_mode='rgb',
+                 dim_ordering=K.image_dim_ordering,
+                 classes=None, class_mode='categorical',
+                 batch_size=32, shuffle=True, seed=None,
+                 save_to_dir=None, save_prefix='', save_format='jpeg'):
+        self.directory = directory
+        self.image_data_generator = image_data_generator
+        self.target_size = tuple(target_size)
+        if color_mode not in {'rgb', 'grayscale'}:
+            raise ValueError('Invalid color mode:', color_mode,
+                             '; expected "rgb" or "grayscale".')
+        self.color_mode = color_mode
+        self.dim_ordering = dim_ordering
+        if self.color_mode == 'rgb':
+            if self.dim_ordering == 'tf':
+                self.image_shape = self.target_size + (3,)
+            else:
+                self.image_shape = (3,) + self.target_size
+        else:
+            if self.dim_ordering == 'tf':
+                self.image_shape = self.target_size + (1,)
+            else:
+                self.image_shape = (1,) + self.target_size
+        self.classes = classes
+        if class_mode not in {'categorical', 'binary', 'sparse', None}:
+            raise ValueError('Invalid class_mode:', class_mode,
+                             '; expected one of "categorical", '
+                             '"binary", "sparse", or None.')
+        self.class_mode = class_mode
+        self.save_to_dir = save_to_dir
+        self.save_prefix = save_prefix
+        self.save_format = save_format
+
+        white_list_formats = {'png', 'jpg', 'jpeg', 'bmp'}
+
+        # first, count the number of samples and classes
+        self.nb_sample = 0
+
+        if not classes:
+            classes = []
+            for subdir in sorted(os.listdir(directory)):
+                if os.path.isdir(os.path.join(directory, subdir)):
+                    classes.append(subdir)
+        self.nb_class = len(classes)
+        self.class_indices = dict(zip(classes, range(len(classes))))
+
+        for subdir in classes:
+            subpath = os.path.join(directory, subdir)
+            for fname in os.listdir(subpath):
+                is_valid = False
+                for extension in white_list_formats:
+                    if fname.lower().endswith('.' + extension):
+                        is_valid = True
+                        break
+                if is_valid:
+                    self.nb_sample += 1
+        print('Found %d images belonging to %d classes.' % (self.nb_sample, self.nb_class))
+
+        # second, build an index of the images in the different class subfolders
+        self.filenames = []
+        self.classes = np.zeros((self.nb_sample,), dtype='int32')
+        i = 0
+        for subdir in classes:
+            subpath = os.path.join(directory, subdir)
+            for fname in os.listdir(subpath):
+                is_valid = False
+                for extension in white_list_formats:
+                    if fname.lower().endswith('.' + extension):
+                        is_valid = True
+                        break
+                if is_valid:
+                    self.classes[i] = self.class_indices[subdir]
+                    self.filenames.append(os.path.join(subdir, fname))
+                    i += 1
+        super(DirectoryIterator, self).__init__(self.nb_sample, batch_size, shuffle, seed)
+
+    def next(self):
+        with self.lock:
+            index_array, current_index, current_batch_size = next(self.index_generator)
+        # The transformation of images is not under thread lock so it can be done in parallel
+        batch_x = np.zeros((current_batch_size,) + self.image_shape)
+        grayscale = self.color_mode == 'grayscale'
+        # build batch of image data
+        for i, j in enumerate(index_array):
+            fname = self.filenames[j]
+            img = load_img(os.path.join(self.directory, fname), grayscale=grayscale, target_size=self.target_size)
+            x = img_to_array(img, dim_ordering=self.dim_ordering)
+            x = self.image_data_generator.random_transform(x)
+            x = self.image_data_generator.standardize(x)
+            batch_x[i] = x
+        # optionally save augmented images to disk for debugging purposes
+        if self.save_to_dir:
+            for i in range(current_batch_size):
+                img = array_to_img(batch_x[i], self.dim_ordering, scale=True)
+                fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
+                                                                  index=current_index + i,
+                                                                  hash=np.random.randint(1e4),
+                                                                  format=self.save_format)
+                img.save(os.path.join(self.save_to_dir, fname))
+        # build batch of labels
+        if self.class_mode == 'sparse':
+            batch_y = self.classes[index_array]
+        elif self.class_mode == 'binary':
+            batch_y = self.classes[index_array].astype('float32')
+        elif self.class_mode == 'categorical':
+            batch_y = np.zeros((len(batch_x), self.nb_class), dtype='float32')
+            for i, label in enumerate(self.classes[index_array]):
+                batch_y[i, label] = 1.
+        else:
+            return batch_x
+        return batch_x, batch_y

keras/preprocessing/sequence.py

@@ -100,7 +100,7 @@ def skipgrams(sequence, vocabulary_size,
     '''Take a sequence (list of indexes of words),
     returns couples of [word_index, other_word index] and labels (1s or 0s),
     where label = 1 if 'other_word' belongs to the context of 'word',
-    and label=0 if 'other_word' is ramdomly sampled
+    and label=0 if 'other_word' is randomly sampled
 
     # Arguments
         vocabulary_size: int. maximum possible word index + 1
@@ -113,7 +113,7 @@ def skipgrams(sequence, vocabulary_size,
             if True labels will be categorical eg. [[1,0],[0,1],[0,1] .. ]
 
     # Returns
-        couples, lables: where `couples` are int pairs and
+        couples, labels: where `couples` are int pairs and
             `labels` are either 0 or 1.
 
     # Notes

keras/preprocessing/text.py

@@ -3,6 +3,7 @@
 from a fast Cython rewrite.
 '''
 from __future__ import absolute_import
+from __future__ import division
 
 import string
 import sys
@@ -206,9 +207,11 @@ class Tokenizer(object):
                 elif mode == 'binary':
                     X[i][j] = 1
                 elif mode == 'tfidf':
-                    tf = np.log(c / len(seq))
-                    df = (1 + np.log(1 + self.index_docs.get(j, 0) / (1 + self.document_count)))
-                    X[i][j] = tf / df
+                    # Use weighting scheme 2 in
+                    #   https://en.wikipedia.org/wiki/Tf%E2%80%93idf
+                    tf = 1 + np.log(c)
+                    idf = np.log(1 + self.document_count / (1 + self.index_docs.get(j, 0)))
+                    X[i][j] = tf * idf
                 else:
                     raise Exception('Unknown vectorization mode: ' + str(mode))
         return X

keras/regularizers.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import numpy as np
 from . import backend as K
 
 
@@ -16,6 +17,47 @@ class Regularizer(object):
         return {'name': self.__class__.__name__}
 
 
+class EigenvalueRegularizer(Regularizer):
+    '''This takes a constant that controls the 
+    regularization by Eigenvalue Decay on the 
+    current layer and outputs the regularized 
+    loss (evaluated on the training data) and 
+    the original loss (evaluated on the 
+    validation data).
+    '''
+    def __init__(self, k):
+        self.k = k
+        self.uses_learning_phase = True
+
+    def set_param(self, p):
+        self.p = p
+
+    def __call__(self, loss):
+        power = 9  # number of iterations of the power method
+        W = self.p
+        if K.ndim(W) > 2:
+            raise Exception('Eigenvalue Decay regularizer '
+                            'is only available for dense '
+                            'and embedding layers.')
+        WW = K.dot(K.transpose(W), W)
+        dim1, dim2 = K.eval(K.shape(WW))  # number of neurons in the layer
+        k = self.k
+        
+        # power method for approximating the dominant eigenvector:
+        o = K.ones([dim1, 1])  # initial values for the dominant eigenvector
+        domin_eigenvect = K.dot(WW, o)
+        for n in range(power - 1):
+            domin_eigenvect = K.dot(WW, domin_eigenvect)    
+        
+        WWd = K.dot(WW, domin_eigenvect)
+        
+        # the corresponding dominant eigenvalue:
+        domin_eigenval = K.dot(K.transpose(WWd), domin_eigenvect) / K.dot(K.transpose(domin_eigenvect), domin_eigenvect)
+        regularized_loss = loss + (domin_eigenval ** 0.5) * self.k  # multiplied by the given regularization gain
+
+        return K.in_train_phase(regularized_loss[0, 0], loss)
+
+
 class WeightRegularizer(Regularizer):
     def __init__(self, l1=0., l2=0.):
         self.l1 = K.cast_to_floatx(l1)
@@ -41,8 +83,8 @@ class WeightRegularizer(Regularizer):
 
     def get_config(self):
         return {'name': self.__class__.__name__,
-                'l1': self.l1,
-                'l2': self.l2}
+                'l1': float(self.l1),
+                'l2': float(self.l2)}
 
 
 class ActivityRegularizer(Regularizer):
@@ -59,15 +101,17 @@ class ActivityRegularizer(Regularizer):
             raise Exception('Need to call `set_layer` on '
                             'ActivityRegularizer instance '
                             'before calling the instance.')
-        output = self.layer.output
-        regularized_loss = loss + self.l1 * K.sum(K.mean(K.abs(output), axis=0))
-        regularized_loss += self.l2 * K.sum(K.mean(K.square(output), axis=0))
+        regularized_loss = loss
+        for i in range(len(self.layer.inbound_nodes)):
+            output = self.layer.get_output_at(i)
+            regularized_loss += self.l1 * K.sum(K.mean(K.abs(output), axis=0))
+            regularized_loss += self.l2 * K.sum(K.mean(K.square(output), axis=0))
         return K.in_train_phase(regularized_loss, loss)
 
     def get_config(self):
         return {'name': self.__class__.__name__,
-                'l1': self.l1,
-                'l2': self.l2}
+                'l1': float(self.l1),
+                'l2': float(self.l2)}
 
 
 def l1(l=0.01):

keras/utils/data_utils.py

@@ -73,7 +73,7 @@ def get_file(fname, origin, untar=False):
         except (Exception, KeyboardInterrupt) as e:
             if os.path.exists(fpath):
                 os.remove(fpath)
-            raise e
+            raise
         progbar = None
 
     if untar:
@@ -88,7 +88,7 @@ def get_file(fname, origin, untar=False):
                         os.remove(untar_fpath)
                     else:
                         shutil.rmtree(untar_fpath)
-                raise e
+                raise
             tfile.close()
         return untar_fpath
 

keras/utils/generic_utils.py

@@ -34,24 +34,28 @@ def make_tuple(*args):
 
 
 class Progbar(object):
-    def __init__(self, target, width=30, verbose=1):
+    def __init__(self, target, width=30, verbose=1, interval=0.01):
         '''
             @param target: total number of steps expected
+            @param interval: minimum visual progress update interval (in seconds)
         '''
         self.width = width
         self.target = target
         self.sum_values = {}
         self.unique_values = []
         self.start = time.time()
+        self.last_update = 0
+        self.interval = interval
         self.total_width = 0
         self.seen_so_far = 0
         self.verbose = verbose
 
-    def update(self, current, values=[]):
+    def update(self, current, values=[], force=False):
         '''
             @param current: index of current step
             @param values: list of tuples (name, value_for_last_step).
             The progress bar will display averages for these values.
+            @param force: force visual progress update
         '''
         for k, v in values:
             if k not in self.sum_values:
@@ -64,6 +68,9 @@ class Progbar(object):
 
         now = time.time()
         if self.verbose == 1:
+            if not force and (now - self.last_update) < self.interval:
+                return
+
             prev_total_width = self.total_width
             sys.stdout.write("\b" * prev_total_width)
             sys.stdout.write("\r")
@@ -127,6 +134,8 @@ class Progbar(object):
                         info += ' %.4e' % avg
                 sys.stdout.write(info + "\n")
 
+        self.last_update = now
+
     def add(self, n, values=[]):
         self.update(self.seen_so_far + n, values)
 

keras/utils/layer_utils.py

@@ -35,9 +35,11 @@ def layer_from_config(config, custom_objects={}):
     return layer_class.from_config(config['config'])
 
 
-def print_summary(layers, relevant_nodes=None):
-    line_length = 100  # total length of printed lines
-    positions = [35, 55, 67, 100]  # absolute positions of log elements in each line
+def print_summary(layers, relevant_nodes=None, line_length=100, positions=[.33, .55, .67, 1.]):
+    # line_length: total length of printed lines
+    # positions: relative or absolute positions of log elements in each line
+    if positions[-1] <= 1:
+        positions = [int(line_length * p) for p in positions]
     # header names for the different log elements
     to_display = ['Layer (type)', 'Output Shape', 'Param #', 'Connected to']
 

keras/utils/np_utils.py

@@ -53,7 +53,7 @@ def categorical_probas_to_classes(p):
 
 
 def convert_kernel(kernel, dim_ordering='th'):
-    '''Converts a kernel matrix (numpy array)
+    '''Converts a kernel matrix (Numpy array)
     from Theano format to TensorFlow format
     (or reciprocally, since the transformation
     is its own inverse).

keras/utils/visualize_util.py

@@ -9,7 +9,7 @@ if not pydot.find_graphviz():
                        ' and graphviz for `pydotprint` to work.')
 
 
-def model_to_dot(model, show_shapes=False):
+def model_to_dot(model, show_shapes=False, show_layer_names=True):
     dot = pydot.Dot()
     dot.set('rankdir', 'TB')
     dot.set('concentrate', True)
@@ -24,19 +24,25 @@ def model_to_dot(model, show_shapes=False):
     # first, populate the nodes of the graph
     for layer in layers:
         layer_id = str(id(layer))
-        label = str(layer.name) + ' (' + layer.__class__.__name__ + ')'
+        if show_layer_names:
+            label = str(layer.name) + ' (' + layer.__class__.__name__ + ')'
+        else:
+            label = layer.__class__.__name__
 
         if show_shapes:
             # Build the label that will actually contain a table with the
             # input/output
-            outputlabels = str(layer.output_shape)
+            try:
+                outputlabels = str(layer.output_shape)
+            except:
+                outputlabels = 'multiple'
             if hasattr(layer, 'input_shape'):
                 inputlabels = str(layer.input_shape)
             elif hasattr(layer, 'input_shapes'):
                 inputlabels = ', '.join(
                     [str(ishape) for ishape in layer.input_shapes])
             else:
-                inputlabels = ''
+                inputlabels = 'multiple'
             label = '%s\n|{input:|output:}|{{%s}|{%s}}' % (label, inputlabels, outputlabels)
 
         node = pydot.Node(layer_id, label=label)
@@ -56,6 +62,6 @@ def model_to_dot(model, show_shapes=False):
     return dot
 
 
-def plot(model, to_file='model.png', show_shapes=False):
-    dot = model_to_dot(model, show_shapes)
+def plot(model, to_file='model.png', show_shapes=False, show_layer_names=True):
+    dot = model_to_dot(model, show_shapes, show_layer_names)
     dot.write_png(to_file)

keras/wrappers/scikit_learn.py

@@ -29,7 +29,7 @@ class BaseWrapper(object):
 
     `sk_params` takes both model parameters and fitting parameters. Legal model
     parameters are the arguments of `build_fn`. Note that like all other
-    estimators in scikit-learn, 'build_fn' should provide defalult values for
+    estimators in scikit-learn, 'build_fn' should provide default values for
     its arguments, so that you could create the estimator without passing any
     values to `sk_params`.
 
@@ -154,10 +154,10 @@ class BaseWrapper(object):
 
         # Arguments
             fn : arbitrary function
-            override: dictionary, values to overrid sk_params
+            override: dictionary, values to override sk_params
 
         # Returns
-            res : dictionary dictionary containing variabls
+            res : dictionary dictionary containing variables
                 in both sk_params and fn's arguments.
         '''
         res = {}
@@ -203,9 +203,19 @@ class KerasClassifier(BaseWrapper):
         # Returns
             proba: array-like, shape `(n_samples, n_outputs)`
                 Class probability estimates.
+                In the case of binary classification,
+                tp match the scikit-learn API,
+                will return an array of shape '(n_samples, 2)'
+                (instead of `(n_sample, 1)` as in Keras).
         '''
         kwargs = self.filter_sk_params(Sequential.predict_proba, kwargs)
-        return self.model.predict_proba(X, **kwargs)
+        probs = self.model.predict_proba(X, **kwargs)
+
+        # check if binary classification
+        if probs.shape[1] == 1:
+            # first column is probability of class 0 and second is of class 1
+            probs = np.hstack([1 - probs, probs])
+        return probs
 
     def score(self, X, y, **kwargs):
         '''Returns the mean accuracy on the given test data and labels.

setup.py

@@ -3,12 +3,12 @@ from setuptools import find_packages
 
 
 setup(name='Keras',
-      version='1.0.1',
+      version='1.0.5',
       description='Deep Learning for Python',
       author='Francois Chollet',
       author_email='francois.chollet@gmail.com',
       url='https://github.com/fchollet/keras',
-      download_url='https://github.com/fchollet/keras/tarball/1.0.1',
+      download_url='https://github.com/fchollet/keras/tarball/1.0.5',
       license='MIT',
       install_requires=['theano', 'pyyaml', 'six'],
       extras_require={

tests/integration_tests/test_temporal_data_tasks.py

@@ -23,7 +23,7 @@ def test_temporal_classification():
     '''
     np.random.seed(1337)
     (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=500,
-                                                         nb_test=200,
+                                                         nb_test=500,
                                                          input_shape=(3, 5),
                                                          classification=True,
                                                          nb_class=2)
@@ -35,12 +35,12 @@ def test_temporal_classification():
                   input_shape=(X_train.shape[1], X_train.shape[2]),
                   activation='softmax'))
     model.compile(loss='categorical_crossentropy',
-                  optimizer='adadelta',
+                  optimizer='adagrad',
                   metrics=['accuracy'])
-    history = model.fit(X_train, y_train, nb_epoch=5, batch_size=16,
+    history = model.fit(X_train, y_train, nb_epoch=20, batch_size=32,
                         validation_data=(X_test, y_test),
                         verbose=0)
-    assert(history.history['val_acc'][-1] > 0.9)
+    assert(history.history['val_acc'][-1] >= 0.85)
 
 
 def test_temporal_regression():
@@ -182,4 +182,5 @@ def test_masked_temporal():
     assert(np.abs(history.history['val_loss'][-1] - ground_truth) < 0.06)
 
 if __name__ == '__main__':
-    pytest.main([__file__])
+    # pytest.main([__file__])
+    test_temporal_classification()

tests/keras/backend/test_backends.py

@@ -38,6 +38,26 @@ def check_two_tensor_operation(function_name, x_input_shape,
     assert zth.shape == ztf.shape
     assert_allclose(zth, ztf, atol=1e-05)
 
+def check_composed_tensor_operations(first_function_name, first_function_args,
+                                     second_function_name, second_function_args,
+                                     input_shape):
+    ''' Creates a random tensor t0 with shape input_shape and compute
+                 t1 = first_function_name(t0, **first_function_args)
+                 t2 = second_function_name(t1, **second_function_args)
+        with both Theano and TensorFlow backends and ensures the answers match.
+    '''
+    val = np.random.random(input_shape) - 0.5
+    xth = KTH.variable(val)
+    xtf = KTF.variable(val)
+
+    yth = getattr(KTH, first_function_name)(xth, **first_function_args)
+    ytf = getattr(KTF, first_function_name)(xtf, **first_function_args)
+
+    zth = KTH.eval(getattr(KTH, second_function_name)(yth, **second_function_args))
+    ztf = KTF.eval(getattr(KTF, second_function_name)(ytf, **second_function_args))
+
+    assert zth.shape == ztf.shape
+    assert_allclose(zth, ztf, atol=1e-05)   
 
 class TestBackend(object):
 
@@ -70,6 +90,9 @@ class TestBackend(object):
         check_single_tensor_operation('expand_dims', (4, 3), dim=-1)
         check_single_tensor_operation('expand_dims', (4, 3, 2), dim=1)
         check_single_tensor_operation('squeeze', (4, 3, 1), axis=2)
+        check_composed_tensor_operations('reshape', {'shape':(4,3,1,1)}, 
+                                         'squeeze', {'axis':2}, 
+                                         (4, 3, 1, 1))
 
     def test_repeat_elements(self):
         reps = 3
@@ -91,6 +114,17 @@ class TestBackend(object):
                 assert_allclose(np_rep, th_rep, atol=1e-05)
                 assert_allclose(np_rep, tf_rep, atol=1e-05)
 
+    def test_tile(self):
+        shape = (3, 4)
+        arr = np.arange(np.prod(shape)).reshape(shape)
+        arr_th = KTH.variable(arr)
+        arr_tf = KTF.variable(arr)
+
+        n = (2, 1)
+        th_rep = KTH.eval(KTH.tile(arr_th, n))
+        tf_rep = KTF.eval(KTF.tile(arr_tf, n))
+        assert_allclose(tf_rep, th_rep, atol=1e-05)
+
     def test_value_manipulation(self):
         val = np.random.random((4, 2))
         xth = KTH.variable(val)
@@ -139,6 +173,9 @@ class TestBackend(object):
         # does not work yet, wait for bool <-> int casting in TF (coming soon)
         # check_single_tensor_operation('any', (4, 2))
         # check_single_tensor_operation('any', (4, 2), axis=1, keepdims=True)
+        #
+        # check_single_tensor_operation('any', (4, 2))
+        # check_single_tensor_operation('any', (4, 2), axis=1, keepdims=True)
 
         check_single_tensor_operation('argmax', (4, 2))
         check_single_tensor_operation('argmax', (4, 2), axis=1)

tests/keras/engine/test_topology.py

@@ -389,6 +389,9 @@ def test_recursion():
         assert K.int_shape(m_tf) == (None, 64)
         assert K.int_shape(n_tf) == (None, 5)
 
+        # test merge
+        o_tf = merge([j_tf, k_tf], mode='concat', concat_axis=1)
+
 
 def test_functional_guide():
     # MNIST
@@ -512,8 +515,6 @@ def test_sequential_regression():
                            name='embed_1'))
     branch_1.add(LSTM(32, name='lstm_1'))
 
-    branch_1.add(BatchNormalization())
-
     branch_2 = Sequential(name='branch_2')
     branch_2.add(Dense(32, input_shape=(8,), name='dense_2'))
 

tests/keras/engine/test_training.py

@@ -117,10 +117,10 @@ def test_model_methods():
 
     out = model.train_on_batch([input_a_np, input_b_np],
                                [output_a_np, output_b_np])
-    assert len(out) == 3
+    assert len(out) == 5
     out = model.test_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np])
-    assert len(out) == 3
+    assert len(out) == 5
 
     # this should also work
     model.compile(optimizer, loss, metrics={'dense_1': 'acc'},
@@ -128,10 +128,10 @@ def test_model_methods():
 
     out = model.train_on_batch([input_a_np, input_b_np],
                                [output_a_np, output_b_np])
-    assert len(out) == 2
+    assert len(out) == 4
     out = model.test_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np])
-    assert len(out) == 2
+    assert len(out) == 4
 
     # and this as well
     model.compile(optimizer, loss, metrics={'dense_1': ['acc']},
@@ -139,10 +139,10 @@ def test_model_methods():
 
     out = model.train_on_batch([input_a_np, input_b_np],
                                [output_a_np, output_b_np])
-    assert len(out) == 2
+    assert len(out) == 4
     out = model.test_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np])
-    assert len(out) == 2
+    assert len(out) == 4
 
     # test with a custom metric function
     mse = lambda y_true, y_pred: K.mean(K.pow(y_true - y_pred, 2))
@@ -151,10 +151,10 @@ def test_model_methods():
 
     out = model.train_on_batch([input_a_np, input_b_np],
                                [output_a_np, output_b_np])
-    assert len(out) == 3
+    assert len(out) == 5
     out = model.test_on_batch([input_a_np, input_b_np],
                               [output_a_np, output_b_np])
-    assert len(out) == 3
+    assert len(out) == 5
 
     input_a_np = np.random.random((10, 3))
     input_b_np = np.random.random((10, 3))

tests/keras/layers/test_core.py

@@ -1,6 +1,5 @@
 import pytest
 import numpy as np
-from numpy.testing import assert_allclose
 
 from keras import backend as K
 from keras.layers import core
@@ -14,19 +13,19 @@ def test_masking():
 
 
 def test_merge():
-    from keras.layers import Input, merge
+    from keras.layers import Input, merge, Merge
     from keras.models import Model
 
     # test modes: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot'.
     input_shapes = [(3, 2), (3, 2)]
     inputs = [np.random.random(shape) for shape in input_shapes]
 
-    # test graph API
-    for mode in ['sum', 'mul', 'concat', 'ave', 'cos', 'dot']:
+    # test functional API
+    for mode in ['sum', 'mul', 'concat', 'ave']:
         print(mode)
         input_a = Input(shape=input_shapes[0][1:])
         input_b = Input(shape=input_shapes[1][1:])
-        merged = merge([input_a, input_b], mode='sum')
+        merged = merge([input_a, input_b], mode=mode)
         model = Model([input_a, input_b], merged)
         model.compile('rmsprop', 'mse')
 
@@ -38,6 +37,15 @@ def test_merge():
         model = Model.from_config(config)
         model.compile('rmsprop', 'mse')
 
+        # test Merge (#2460)
+        merged = Merge(mode=mode)([input_a, input_b])
+        model = Model([input_a, input_b], merged)
+        model.compile('rmsprop', 'mse')
+
+        expected_output_shape = model.get_output_shape_for(input_shapes)
+        actual_output_shape = model.predict(inputs).shape
+        assert expected_output_shape == actual_output_shape
+
     # test lambda with output_shape lambda
     input_a = Input(shape=input_shapes[0][1:])
     input_b = Input(shape=input_shapes[1][1:])
@@ -75,6 +83,60 @@ def test_merge():
     model.compile('rmsprop', 'mse')
 
 
+def test_merge_mask_2d():
+    from keras.layers import Input, merge, Masking
+    from keras.models import Model
+
+    rand = lambda *shape: np.asarray(np.random.random(shape) > 0.5, dtype='int32')
+
+    # inputs
+    input_a = Input(shape=(3,))
+    input_b = Input(shape=(3,))
+
+    # masks
+    masked_a = Masking(mask_value=0)(input_a)
+    masked_b = Masking(mask_value=0)(input_b)
+
+    # two different types of merging
+    merged_sum = merge([masked_a, masked_b], mode='sum')
+    merged_concat = merge([masked_a, masked_b], mode='concat', concat_axis=1)
+
+    # test sum
+    model_sum = Model([input_a, input_b], [merged_sum])
+    model_sum.compile(loss='mse', optimizer='sgd')
+    model_sum.fit([rand(2,3), rand(2,3)], [rand(2,3)], nb_epoch=1)
+
+    # test concatenation
+    model_concat = Model([input_a, input_b], [merged_concat])
+    model_concat.compile(loss='mse', optimizer='sgd')
+    model_concat.fit([rand(2,3), rand(2,3)], [rand(2,6)], nb_epoch=1)
+
+
+def test_merge_mask_3d():
+    from keras.layers import Input, merge, Embedding, SimpleRNN
+    from keras.models import Model
+
+    rand = lambda *shape: np.asarray(np.random.random(shape) > 0.5, dtype='int32')
+
+    # embeddings
+    input_a = Input(shape=(3,), dtype='int32')
+    input_b = Input(shape=(3,), dtype='int32')
+    embedding = Embedding(3, 4, mask_zero=True)
+    embedding_a = embedding(input_a)
+    embedding_b = embedding(input_b)
+
+    # rnn
+    rnn = SimpleRNN(3, return_sequences=True)
+    rnn_a = rnn(embedding_a)
+    rnn_b = rnn(embedding_b)
+
+    # concatenation
+    merged_concat = merge([rnn_a, rnn_b], mode='concat', concat_axis=-1)
+    model = Model([input_a, input_b], [merged_concat])
+    model.compile(loss='mse', optimizer='sgd')
+    model.fit([rand(2,3), rand(2,3)], [rand(2,3,6)])
+
+
 def test_dropout():
     layer_test(core.Dropout,
                kwargs={'p': 0.5},

tests/keras/layers/test_normalization.py

@@ -24,22 +24,22 @@ def basic_batchnorm_test():
                input_shape=(3, 4, 2))
 
 
-def test_batchnorm_mode_0():
-    model = Sequential()
-    norm_m0 = normalization.BatchNormalization(mode=0, input_shape=(10,))
-    model.add(norm_m0)
-    model.compile(loss='mse', optimizer='sgd')
+def test_batchnorm_mode_0_or_2():
+    for mode in [0, 2]:
+        model = Sequential()
+        norm_m0 = normalization.BatchNormalization(mode=mode, input_shape=(10,))
+        model.add(norm_m0)
+        model.compile(loss='mse', optimizer='sgd')
 
-    # centered on 5.0, variance 10.0
-    X = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10))
-    model.fit(X, X, nb_epoch=5, verbose=0)
-    out = norm_m0.call(K.variable(X))
-    out -= norm_m0.beta
-    out /= norm_m0.gamma
-    np_out = K.function([K.learning_phase()], [out])([1.])[0]
+        # centered on 5.0, variance 10.0
+        X = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10))
+        model.fit(X, X, nb_epoch=5, verbose=0)
+        out = model.predict(X)
+        out -= K.eval(norm_m0.beta)
+        out /= K.eval(norm_m0.gamma)
 
-    assert_allclose(np_out.mean(), 0.0, atol=1e-1)
-    assert_allclose(np_out.std(), 1.0, atol=1e-1)
+        assert_allclose(out.mean(), 0.0, atol=1e-1)
+        assert_allclose(out.std(), 1.0, atol=1e-1)
 
 
 def test_batchnorm_mode_0_convnet():
@@ -51,13 +51,12 @@ def test_batchnorm_mode_0_convnet():
     # centered on 5.0, variance 10.0
     X = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
     model.fit(X, X, nb_epoch=5, verbose=0)
-    out = norm_m0.call(K.variable(X))
-    out -= K.reshape(norm_m0.beta, (1, 3, 1, 1))
-    out /= K.reshape(norm_m0.gamma, (1, 3, 1, 1))
-    np_out = K.function([K.learning_phase()], [out])([1.])[0]
+    out = model.predict(X)
+    out -= np.reshape(K.eval(norm_m0.beta), (1, 3, 1, 1))
+    out /= np.reshape(K.eval(norm_m0.gamma), (1, 3, 1, 1))
 
-    assert_allclose(np.mean(np_out, axis=(0, 2, 3)), 0.0, atol=1e-1)
-    assert_allclose(np.std(np_out, axis=(0, 2, 3)), 1.0, atol=1e-1)
+    assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1)
+    assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1)
 
 
 def test_batchnorm_mode_1():

tests/keras/layers/test_recurrent.py

@@ -32,6 +32,13 @@ def _runner(layer_class):
                        'dropout_W': 0.1},
                input_shape=(3, 2, 3))
 
+    # check implementation modes
+    for mode in ['cpu', 'mem', 'gpu']:
+        layer_test(layer_class,
+                   kwargs={'output_dim': output_dim,
+                           'consume_less': mode},
+                   input_shape=(3, 2, 3))
+
     # check statefulness
     model = Sequential()
     model.add(embeddings.Embedding(embedding_num, embedding_dim,

tests/keras/preprocessing/test_image.py

@@ -4,85 +4,91 @@ from PIL import Image
 import numpy as np
 import os
 import shutil
+import tempfile
 
 
-def setup_function(func):
-    paths = ['test_images', 'test_images/rgb', 'test_images/gsc']
-    for path in paths:
-        if not os.path.exists(path):
-            os.mkdir(path)
+class TestImage:
 
-    img_w = img_h = 20
-    for n in range(8):
-        bias = np.random.rand(img_w, img_h, 1) * 64
-        variance = np.random.rand(img_w, img_h, 1) * (255-64)
-        imarray = np.random.rand(img_w, img_h, 3) * variance + bias
-        im = Image.fromarray(imarray.astype('uint8')).convert('RGBA')
-        im.save('test_images/rgb/rgb_test_image_'+str(n)+'.png')
+    def setup_class(cls):
+        img_w = img_h = 20
+        rgb_images = []
+        gray_images = []
+        for n in range(8):
+            bias = np.random.rand(img_w, img_h, 1) * 64
+            variance = np.random.rand(img_w, img_h, 1) * (255-64)
+            imarray = np.random.rand(img_w, img_h, 3) * variance + bias
+            im = Image.fromarray(imarray.astype('uint8')).convert('RGB')
+            rgb_images.append(im)
 
-        imarray = np.random.rand(img_w, img_h, 1) * variance + bias
-        im = Image.fromarray(imarray.astype('uint8').squeeze()).convert('L')
-        im.save('test_images/gsc/gsc_test_image_'+str(n)+'.png')
+            imarray = np.random.rand(img_w, img_h, 1) * variance + bias
+            im = Image.fromarray(imarray.astype('uint8').squeeze()).convert('L')
+            gray_images.append(im)
 
+        cls.all_test_images = [rgb_images, gray_images]
 
-def teardown_function(func):
-    shutil.rmtree('test_images')
+    def teardown_class(cls):
+        del cls.all_test_images
 
+    def test_image_data_generator(self):
+        for test_images in self.all_test_images:
+            img_list = []
+            for im in test_images:
+                img_list.append(img_to_array(im)[None, ...])
 
-def test_image_data_generator():
-    for color_mode in ['gsc', 'rgb']:
-        file_list = list_pictures('test_images/' + color_mode)
-        img_list = []
-        for f in file_list:
-            img_list.append(img_to_array(load_img(f))[None, ...])
+            images = np.vstack(img_list)
+            generator = ImageDataGenerator(
+                featurewise_center=True,
+                samplewise_center=True,
+                featurewise_std_normalization=True,
+                samplewise_std_normalization=True,
+                zca_whitening=True,
+                rotation_range=90.,
+                width_shift_range=0.1,
+                height_shift_range=0.1,
+                shear_range=0.5,
+                zoom_range=0.2,
+                channel_shift_range=0.,
+                fill_mode='nearest',
+                cval=0.5,
+                horizontal_flip=True,
+                vertical_flip=True)
+            generator.fit(images, augment=True)
 
-        images = np.vstack(img_list)
-        generator = ImageDataGenerator(
-            featurewise_center=True,
-            samplewise_center=True,
-            featurewise_std_normalization=True,
-            samplewise_std_normalization=True,
-            zca_whitening=True,
-            rotation_range=90.,
-            width_shift_range=10.,
-            height_shift_range=10.,
-            shear_range=0.5,
-            horizontal_flip=True,
-            vertical_flip=True)
-        generator.fit(images, augment=True)
+            tmp_folder = tempfile.mkdtemp(prefix='test_images')
+            for x, y in generator.flow(images, np.arange(images.shape[0]),
+                                       shuffle=True, save_to_dir=tmp_folder):
+                assert x.shape[1:] == images.shape[1:]
+                break
+            shutil.rmtree(tmp_folder)
 
-        for x, y in generator.flow(images, np.arange(images.shape[0]),
-                                   shuffle=True, save_to_dir='test_images'):
-            assert x.shape[1:] == images.shape[1:]
-            break
+    def test_img_flip(self):
+        x = np.array(range(4)).reshape([1, 1, 2, 2])
+        assert (flip_axis(x, 0) == x).all()
+        assert (flip_axis(x, 1) == x).all()
+        assert (flip_axis(x, 2) == [[[[2, 3], [0, 1]]]]).all()
+        assert (flip_axis(x, 3) == [[[[1, 0], [3, 2]]]]).all()
 
-
-def test_img_flip():
-    x = np.array(range(4)).reshape([1, 1, 2, 2])
-    assert (flip_axis(x, 0) == x).all()
-    assert (flip_axis(x, 1) == x).all()
-    assert (flip_axis(x, 2) == [[[[2, 3], [0, 1]]]]).all()
-    assert (flip_axis(x, 3) == [[[[1, 0], [3, 2]]]]).all()
-
-    dim_ordering_and_col_index = (('tf', 2), ('th', 3))
-    for dim_ordering, col_index in dim_ordering_and_col_index:
-        image_generator_th = ImageDataGenerator(
-            featurewise_center=False,
-            samplewise_center=False,
-            featurewise_std_normalization=False,
-            samplewise_std_normalization=False,
-            zca_whitening=False,
-            rotation_range=0,
-            width_shift_range=0,
-            height_shift_range=0,
-            shear_range=0,
-            horizontal_flip=True,
-            vertical_flip=False,
-            dim_ordering=dim_ordering).flow(x, [1])
-        for i in range(10):
-            potentially_flipped_x, _ = next(image_generator_th)
-            assert ((potentially_flipped_x == x).all() or
-                    (potentially_flipped_x == flip_axis(x, col_index)).all())
+        dim_ordering_and_col_index = (('tf', 2), ('th', 3))
+        for dim_ordering, col_index in dim_ordering_and_col_index:
+            image_generator_th = ImageDataGenerator(
+                featurewise_center=False,
+                samplewise_center=False,
+                featurewise_std_normalization=False,
+                samplewise_std_normalization=False,
+                zca_whitening=False,
+                rotation_range=0,
+                width_shift_range=0,
+                height_shift_range=0,
+                shear_range=0,
+                zoom_range=0,
+                channel_shift_range=0,
+                horizontal_flip=True,
+                vertical_flip=False,
+                dim_ordering=dim_ordering).flow(x, [1])
+            for i in range(10):
+                potentially_flipped_x, _ = next(image_generator_th)
+                assert ((potentially_flipped_x == x).all() or
+                        (potentially_flipped_x == flip_axis(x, col_index)).all())
 
 
 if __name__ == '__main__':

tests/keras/test_activations.py

@@ -56,6 +56,22 @@ def test_softplus():
     assert_allclose(result, expected, rtol=1e-05)
 
 
+def test_softsign():
+    '''
+    Test using a reference softsign implementation
+    '''
+    def softsign(x):
+        return np.divide(x, np.ones_like(x) + np.absolute(x))
+
+    x = K.placeholder(ndim=2)
+    f = K.function([x],  [activations.softsign(x)])
+    test_values = get_standard_values()
+
+    result = f([test_values])[0]
+    expected = softsign(test_values)
+    assert_allclose(result, expected, rtol=1e-05)
+
+
 def test_sigmoid():
     '''
     Test using a numerically stable reference sigmoid implementation

tests/keras/test_callbacks.py

@@ -105,6 +105,27 @@ def test_EarlyStopping():
                         validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=20)
 
 
+def test_EarlyStopping_reuse():
+    patience = 3
+    data = np.random.random((100, 1))
+    labels = np.where(data > 0.5, 1, 0)
+    model = Sequential((
+        Dense(1, input_dim=1, activation='relu'),
+        Dense(1, activation='sigmoid'),
+    ))
+    model.compile(optimizer='sgd', loss='binary_crossentropy', metrics=['accuracy'])
+    stopper = callbacks.EarlyStopping(monitor='acc', patience=patience)
+    weights = model.get_weights()
+
+    hist = model.fit(data, labels, callbacks=[stopper])
+    assert len(hist.epoch) >= patience
+
+    # This should allow training to go for at least `patience` epochs
+    model.set_weights(weights)
+    hist = model.fit(data, labels, callbacks=[stopper])
+    assert len(hist.epoch) >= patience
+
+
 def test_LearningRateScheduler():
     (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples,
                                                          nb_test=test_samples,

tests/keras/test_graph_model.py

@@ -93,7 +93,7 @@ def test_1o_1i():
     assert(len(out) == 1)
     loss = graph.test_on_batch({'input1': X_test_graph, 'output1': y_test_graph})
     loss = graph.train_on_batch({'input1': X_test_graph, 'output1': y_test_graph})
-    loss = graph.evaluate({'input1': X_test_graph, 'output1': y_test_graph}, verbose=0) 
+    loss = graph.evaluate({'input1': X_test_graph, 'output1': y_test_graph}, verbose=0)
 
     # test accuracy:
     graph.compile('rmsprop', {'output1': 'mse'}, metrics=['accuracy'])
@@ -209,7 +209,7 @@ def test_siamese_1():
     loss = graph.test_on_batch({'input1': X_test_graph, 'input2': X2_test_graph, 'output1': y_test_graph})
     loss = graph.train_on_batch({'input1': X_test_graph, 'input2': X2_test_graph, 'output1': y_test_graph})
     loss = graph.evaluate({'input1': X_test_graph, 'input2': X2_test_graph, 'output1': y_test_graph})
-    assert(loss < 3.0)
+    assert(loss < 4.0)
 
     # test serialization
     config = graph.get_config()

tests/keras/test_initializations.py

@@ -4,8 +4,14 @@ import numpy as np
 from keras import initializations
 from keras import backend as K
 
-SHAPE = (100, 100)
+# 2D tensor test fixture
+FC_SHAPE = (100, 100)
 
+# 4D convolution in th order. This shape has the same effective shape as FC_SHAPE
+CONV_SHAPE = (25, 25, 2, 2)
+
+# The equivalent shape of both test fixtures
+SHAPE = (100, 100)
 
 def _runner(init, shape, target_mean=None, target_std=None,
             target_max=None, target_min=None):
@@ -22,62 +28,78 @@ def _runner(init, shape, target_mean=None, target_std=None,
         assert abs(output.min() - target_min) < lim
 
 
-def test_uniform():
-    _runner(initializations.uniform, SHAPE, target_mean=0.,
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_uniform(tensor_shape):
+    _runner(initializations.uniform, tensor_shape, target_mean=0.,
             target_max=0.05, target_min=-0.05)
 
 
-def test_normal():
-    _runner(initializations.normal, SHAPE, target_mean=0., target_std=0.05)
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_normal(tensor_shape):
+    _runner(initializations.normal, tensor_shape, target_mean=0., target_std=0.05)
 
 
-def test_lecun_uniform():
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_lecun_uniform(tensor_shape):
     scale = np.sqrt(3. / SHAPE[0])
-    _runner(initializations.lecun_uniform, SHAPE,
+    _runner(initializations.lecun_uniform, tensor_shape,
             target_mean=0., target_max=scale, target_min=-scale)
 
 
-def test_glorot_uniform():
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_glorot_uniform(tensor_shape):
     scale = np.sqrt(6. / (SHAPE[0] + SHAPE[1]))
-    _runner(initializations.glorot_uniform, SHAPE, target_mean=0.,
+    _runner(initializations.glorot_uniform, tensor_shape, target_mean=0.,
             target_max=scale, target_min=-scale)
 
 
-def test_glorot_normal():
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_glorot_normal(tensor_shape):
     scale = np.sqrt(2. / (SHAPE[0] + SHAPE[1]))
-    _runner(initializations.glorot_normal, SHAPE,
+    _runner(initializations.glorot_normal, tensor_shape,
             target_mean=0., target_std=scale)
 
 
-def test_he_uniform():
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_he_uniform(tensor_shape):
     scale = np.sqrt(6. / SHAPE[0])
-    _runner(initializations.he_uniform, SHAPE, target_mean=0.,
+    _runner(initializations.he_uniform, tensor_shape, target_mean=0.,
             target_max=scale, target_min=-scale)
 
 
-def test_he_normal():
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_he_normal(tensor_shape):
     scale = np.sqrt(2. / SHAPE[0])
-    _runner(initializations.he_normal, SHAPE,
+    _runner(initializations.he_normal, tensor_shape,
             target_mean=0., target_std=scale)
 
 
-def test_orthogonal():
-    _runner(initializations.orthogonal, SHAPE,
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_orthogonal(tensor_shape):
+    _runner(initializations.orthogonal, tensor_shape,
             target_mean=0.)
 
 
-def test_identity():
-    _runner(initializations.identity, SHAPE,
-            target_mean=1./SHAPE[0], target_max=1.)
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_identity(tensor_shape):
+    if len(tensor_shape) > 2:
+        with pytest.raises(Exception):
+            _runner(initializations.identity, tensor_shape,
+                    target_mean=1./SHAPE[0], target_max=1.)
+    else:
+        _runner(initializations.identity, tensor_shape,
+                target_mean=1./SHAPE[0], target_max=1.)
 
 
-def test_zero():
-    _runner(initializations.zero, SHAPE,
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_zero(tensor_shape):
+    _runner(initializations.zero, tensor_shape,
             target_mean=0., target_max=0.)
 
 
-def test_one():
-    _runner(initializations.one, SHAPE,
+@pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV'])
+def test_one(tensor_shape):
+    _runner(initializations.one, tensor_shape,
             target_mean=1., target_max=1.)
 
 

tests/keras/test_metrics.py

@@ -0,0 +1,44 @@
+import pytest
+import numpy as np
+
+from keras import metrics
+from keras import backend as K
+
+all_metrics = [
+    metrics.binary_accuracy,
+    metrics.categorical_accuracy,
+    metrics.mean_squared_error,
+    metrics.mean_absolute_error,
+    metrics.mean_absolute_percentage_error,
+    metrics.mean_squared_logarithmic_error,
+    metrics.squared_hinge,
+    metrics.hinge,
+    metrics.categorical_crossentropy,
+    metrics.binary_crossentropy,
+    metrics.poisson,
+    metrics.cosine_proximity,
+]
+
+all_sparse_metrics = [
+    metrics.sparse_categorical_accuracy,
+    metrics.sparse_categorical_crossentropy,
+]
+
+
+def test_metrics():
+    y_a = K.variable(np.random.random((6, 7)))
+    y_b = K.variable(np.random.random((6, 7)))
+    for metric in all_metrics:
+        output = metric(y_a, y_b)
+        assert K.eval(output).shape == ()
+
+
+def test_sparse_metrics():
+    for metric in all_sparse_metrics:
+        y_a = K.variable(np.random.randint(0, 7, (6,)), dtype=K.floatx())
+        y_b = K.variable(np.random.random((6, 7)), dtype=K.floatx())
+        assert K.eval(metric(y_a, y_b)).shape == ()
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])

tests/keras/test_objectives.py

@@ -12,6 +12,7 @@ allobj = [objectives.mean_squared_error,
           objectives.squared_hinge,
           objectives.hinge, objectives.categorical_crossentropy,
           objectives.binary_crossentropy,
+          objectives.kullback_leibler_divergence,
           objectives.poisson,
           objectives.cosine_proximity]
 

tests/keras/test_optimizers.py

@@ -2,7 +2,7 @@ from __future__ import print_function
 import pytest
 
 from keras.utils.test_utils import get_test_data
-from keras.optimizers import SGD, RMSprop, Adagrad, Adadelta, Adam, Adamax
+from keras.optimizers import SGD, RMSprop, Adagrad, Adadelta, Adam, Adamax, Nadam
 from keras.models import Sequential
 from keras.layers.core import Dense, Activation
 from keras.utils.np_utils import to_categorical
@@ -26,7 +26,7 @@ def get_model(input_dim, nb_hidden, output_dim):
     return model
 
 
-def _test_optimizer(optimizer, target=0.9):
+def _test_optimizer(optimizer, target=0.89):
     model = get_model(X_train.shape[1], 10, y_train.shape[1])
     model.compile(loss='categorical_crossentropy',
                   optimizer=optimizer,
@@ -35,7 +35,7 @@ def _test_optimizer(optimizer, target=0.9):
                         validation_data=(X_test, y_test), verbose=2)
     config = optimizer.get_config()
     assert type(config) == dict
-    assert history.history['val_acc'][-1] > target
+    assert history.history['val_acc'][-1] >= target
 
 
 def test_sgd():
@@ -63,5 +63,9 @@ def test_adamax():
     _test_optimizer(Adamax())
 
 
+def test_nadam():
+    _test_optimizer(Nadam())
+
+
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/test_regularizers.py

@@ -22,19 +22,23 @@ high_weight = 5
 max_train_samples = 5000
 max_test_samples = 1000
 
-# the data, shuffled and split between tran and test sets
-(X_train, y_train), (X_test, y_test) = mnist.load_data()
-X_train = X_train.reshape(60000, 784)[:max_train_samples]
-X_test = X_test.reshape(10000, 784)[:max_test_samples]
-X_train = X_train.astype("float32") / 255
-X_test = X_test.astype("float32") / 255
 
-# convert class vectors to binary class matrices
-y_train = y_train[:max_train_samples]
-y_test = y_test[:max_test_samples]
-Y_train = np_utils.to_categorical(y_train, nb_classes)
-Y_test = np_utils.to_categorical(y_test, nb_classes)
-test_ids = np.where(y_test == np.array(weighted_class))[0]
+def get_data():
+    # the data, shuffled and split between tran and test sets
+    (X_train, y_train), (X_test, y_test) = mnist.load_data()
+    X_train = X_train.reshape(60000, 784)[:max_train_samples]
+    X_test = X_test.reshape(10000, 784)[:max_test_samples]
+    X_train = X_train.astype("float32") / 255
+    X_test = X_test.astype("float32") / 255
+
+    # convert class vectors to binary class matrices
+    y_train = y_train[:max_train_samples]
+    y_test = y_test[:max_test_samples]
+    Y_train = np_utils.to_categorical(y_train, nb_classes)
+    Y_test = np_utils.to_categorical(y_test, nb_classes)
+    test_ids = np.where(y_test == np.array(weighted_class))[0]
+
+    return (X_train, Y_train), (X_test, Y_test), test_ids
 
 
 def create_model(weight_reg=None, activity_reg=None):
@@ -47,7 +51,17 @@ def create_model(weight_reg=None, activity_reg=None):
     return model
 
 
+def test_Eigenvalue_reg():
+    (X_train, Y_train), (X_test, Y_test), test_ids = get_data()
+    reg = regularizers.EigenvalueRegularizer(0.01)
+    model = create_model(weight_reg=reg)
+    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
+    model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
+    model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)
+    
+    
 def test_W_reg():
+    (X_train, Y_train), (X_test, Y_test), test_ids = get_data()
     for reg in [regularizers.l1(),
                 regularizers.l2(),
                 regularizers.l1l2()]:
@@ -59,6 +73,7 @@ def test_W_reg():
 
 
 def test_A_reg():
+    (X_train, Y_train), (X_test, Y_test), test_ids = get_data()
     for reg in [regularizers.activity_l1(), regularizers.activity_l2()]:
         model = create_model(activity_reg=reg)
         model.compile(loss='categorical_crossentropy', optimizer='rmsprop')