Prepare PyPI release.

integer division for python3

.travis.yml

@@ -3,18 +3,18 @@ dist: trusty
 language: python
 matrix:
     include:
-        - python: 3.4
-          env: KERAS_BACKEND=theano
-        - python: 3.4
-          env: KERAS_BACKEND=tensorflow
         - python: 2.7
-          env: KERAS_BACKEND=theano
-        - python: 2.7
-          env: KERAS_BACKEND=tensorflow
+          env: KERAS_BACKEND=theano TEST_MODE=PEP8
         - python: 2.7
           env: KERAS_BACKEND=theano TEST_MODE=INTEGRATION_TESTS
         - python: 2.7
-          env: KERAS_BACKEND=theano TEST_MODE=PEP8
+          env: KERAS_BACKEND=tensorflow
+        - python: 3.5
+          env: KERAS_BACKEND=tensorflow
+        - python: 2.7
+          env: KERAS_BACKEND=theano
+        - python: 3.5
+          env: KERAS_BACKEND=theano
 install:
   # code below is taken from http://conda.pydata.org/docs/travis.html
   # We do this conditionally because it saves us some downloading if the
@@ -34,24 +34,22 @@ install:
 
   - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib pandas pytest h5py
   - source activate test-environment
-  - pip install pytest-cov python-coveralls pytest-xdist coverage==3.7.1 #we need this version of coverage for coveralls.io to work
-  - pip install pep8 pytest-pep8
   - pip install git+git://github.com/Theano/Theano.git
 
   # install PIL for preprocessing tests
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
       conda install pil;
-    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]]; then
+    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.5" ]]; then
       conda install Pillow;
     fi
 
-  - python setup.py install
+  - pip install -e .[tests]
 
   # install TensorFlow
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl;
-    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl;
+      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.1-cp27-none-linux_x86_64.whl;
+    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.5" ]]; then
+      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl;
     fi
 # command to run tests
 script:

CONTRIBUTING.md

@@ -43,7 +43,7 @@ We love pull requests. Here's a quick guide:
 4. Write tests. Your code should have full unit test coverage. If you want to see your PR merged promptly, this is crucial.
 
 5. Run our test suite locally. It's easy: from the Keras folder, simply run: `py.test tests/`.
-  - You will need to install `pytest`, `coveralls`, `pytest-cov`, `pytest-xdist`: `pip install pytest pytest-cov python-coveralls pytest-xdist pep8 pytest-pep8`
+  - You will need to install the test requirements as well: `pip install -e .[tests]`.
 
 6. Make sure all tests are passing:
   - with the Theano backend, on Python 2.7 and Python 3.5

ISSUE_TEMPLATE.md

@@ -1,9 +1,13 @@
-Please make sure that the boxes below are checked before you submit your issue. Thank you!
+Please make sure that the boxes below are checked before you submit your issue. If your issue is an implementation question, please ask your question on [StackOverflow](http://stackoverflow.com/questions/tagged/keras) or [join the Keras Slack channel](https://keras-slack-autojoin.herokuapp.com/) and ask there instead of filing a GitHub issue.
+
+Thank you!
 
 - [ ] Check that you are up-to-date with the master branch of Keras. You can update with:
 pip install git+git://github.com/fchollet/keras.git --upgrade --no-deps
 
+- [ ] If running on TensorFlow, check that you are up-to-date with the latest version. The installation instructions can be found [here](https://www.tensorflow.org/get_started/os_setup).
+
 - [ ] If running on Theano, check that you are up-to-date with the master branch of Theano. You can update with:
 pip install git+git://github.com/Theano/Theano.git --upgrade --no-deps
 
-- [ ] Provide a link to a GitHub Gist of a Python script that can reproduce your issue (or just copy the script here if it is short).
+- [ ] Provide a link to a GitHub Gist of a Python script that can reproduce your issue (or just copy the script here if it is short).

README.md

@@ -1,14 +1,12 @@
 # Keras: Deep Learning library for TensorFlow and Theano
 
 [![Build Status](https://travis-ci.org/fchollet/keras.svg?branch=master)](https://travis-ci.org/fchollet/keras)
-[![PyPI version](https://badge.fury.io/py/keras.svg)](https://badge.fury.io/py/keras)
 [![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/fchollet/keras/blob/master/LICENSE)
-[![Join the chat at https://gitter.im/Keras-io/Lobby](https://badges.gitter.im/Keras-io/Lobby.svg)](https://gitter.im/Keras-io/Lobby)
 
 
 ## You have just found Keras.
 
-Keras is a minimalist, highly modular neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
+Keras is a high-level neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
 
 Use Keras if you need a deep learning library that:
 
@@ -57,9 +55,9 @@ Stacking layers is as easy as `.add()`:
 from keras.layers import Dense, Activation
 
 model.add(Dense(output_dim=64, input_dim=100))
-model.add(Activation("relu"))
+model.add(Activation('relu'))
 model.add(Dense(output_dim=10))
-model.add(Activation("softmax"))
+model.add(Activation('softmax'))
 ```
 
 Once your model looks good, configure its learning process with `.compile()`:
@@ -120,7 +118,7 @@ Keras uses the following dependencies:
 *When using the TensorFlow backend:*
 
 - TensorFlow
-    - [See installation instructions](https://github.com/tensorflow/tensorflow#download-and-setup).
+    - [See installation instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/g3doc/get_started/os_setup.md)
 
 *When using the Theano backend:*
 
@@ -149,9 +147,12 @@ By default, Keras will use TensorFlow as its tensor manipulation library. [Follo
 
 ## Support
 
-You can ask questions and join the development discussion on the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+You can ask questions and join the development discussion:
 
-You can also post bug reports and feature requests in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
+- On the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+- On the [Keras Slack channel](https://kerasteam.slack.com). Use [this link](https://keras-slack-autojoin.herokuapp.com/) to request an invitation to the channel.
+
+You can also post **bug reports and feature requests** (only) in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
 
 
 ------------------

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:7.5-cudnn5-devel
+FROM nvidia/cuda:8.0-cudnn5-devel
 
 ENV CONDA_DIR /opt/conda
 ENV PATH $CONDA_DIR/bin:$PATH
@@ -24,10 +24,10 @@ RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \
 USER keras
 
 # Python
-ARG python_version=3.5.1
-ARG tensorflow_version=0.9.0rc0-cp35-cp35m
+ARG python_version=3.5.2
+ARG tensorflow_version=0.12.0rc0-cp35-cp35m
 RUN conda install -y python=${python_version} && \
-    pip install https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-${tensorflow_version}-linux_x86_64.whl && \
+    pip install https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${tensorflow_version}-linux_x86_64.whl && \
     pip install git+git://github.com/Theano/Theano.git && \
     pip install ipdb pytest pytest-cov python-coveralls coverage==3.7.1 pytest-xdist pep8 pytest-pep8 pydot_ng && \
     conda install Pillow scikit-learn notebook pandas matplotlib nose pyyaml six h5py && \

docs/autogen.py

@@ -40,6 +40,7 @@ Index
     Sequence preprocessing
 
 Objectives
+Metrics
 Optimizers
 Activations
 Callbacks
@@ -79,10 +80,16 @@ from keras import callbacks
 from keras import models
 from keras.engine import topology
 from keras import objectives
+from keras import metrics
 from keras import backend
 from keras import constraints
 from keras import activations
 from keras import regularizers
+from keras.utils import data_utils
+from keras.utils import io_utils
+from keras.utils import layer_utils
+from keras.utils import np_utils
+from keras.utils import generic_utils
 
 
 EXCLUDE = {
@@ -133,8 +140,6 @@ PAGES = [
             core.Dense,
             core.Activation,
             core.Dropout,
-            core.SpatialDropout2D,
-            core.SpatialDropout3D,
             core.Flatten,
             core.Reshape,
             core.Permute,
@@ -145,7 +150,6 @@ PAGES = [
             core.Masking,
             core.Highway,
             core.MaxoutDense,
-            core.TimeDistributedDense,
         ],
     },
     {
@@ -158,6 +162,9 @@ PAGES = [
             convolutional.SeparableConvolution2D,
             convolutional.Deconvolution2D,
             convolutional.Convolution3D,
+            convolutional.Cropping1D,
+            convolutional.Cropping2D,
+            convolutional.Cropping3D,
             convolutional.UpSampling1D,
             convolutional.UpSampling2D,
             convolutional.UpSampling3D,
@@ -221,7 +228,10 @@ PAGES = [
         'page': 'layers/wrappers.md',
         'all_module_classes': [wrappers],
     },
-
+    {
+        'page': 'metrics.md',
+        'all_module_functions': [metrics],
+    },
     {
         'page': 'optimizers.md',
         'all_module_classes': [optimizers],
@@ -234,6 +244,33 @@ PAGES = [
         'page': 'backend.md',
         'all_module_functions': [backend],
     },
+    {
+        'page': 'utils/data_utils.md',
+        'functions': [
+            data_utils.get_file,
+        ]
+    },
+    {
+        'page': 'utils/io_utils.md',
+        'classes': [
+            io_utils.HDF5Matrix
+        ],
+    },
+    {
+        'page': 'utils/layer_utils.md',
+        'functions': [
+            layer_utils.layer_from_config,
+        ]
+    },
+    {
+        'page': 'utils/np_utils.md',
+        'all_module_functions': [np_utils]
+    },
+    {
+        'page': 'utils/generic_utils.md',
+        'all_module_functions': [generic_utils],
+        'classes': [generic_utils.CustomObjectScope]
+    },
 ]
 
 ROOT = 'http://keras.io/'
@@ -281,7 +318,7 @@ def get_function_signature(function, method=True):
     for a in args:
         st += str(a) + ', '
     for a, v in kwargs:
-        if type(v) == str:
+        if isinstance(v, str):
             v = '\'' + v + '\''
         st += str(a) + '=' + str(v) + ', '
     if kwargs or args:

docs/mkdocs.yml

@@ -38,6 +38,7 @@ pages:
   - Text Preprocessing: preprocessing/text.md
   - Image Preprocessing: preprocessing/image.md
 - Objectives: objectives.md
+- Metrics: metrics.md
 - Optimizers: optimizers.md
 - Activations: activations.md
 - Callbacks: callbacks.md
@@ -49,7 +50,12 @@ pages:
 - Constraints: constraints.md
 - Visualization: visualization.md
 - Scikit-learn API: scikit-learn-api.md
-
+- Utils:
+  - Data Utils: utils/data_utils.md
+  - I/O Utils: utils/io_utils.md
+  - Layer Utils: utils/layer_utils.md
+  - Numpy Utils: utils/np_utils.md
+  - Generic Utils: utils/generic_utils.md
 
 
 

docs/templates/applications.md

@@ -7,18 +7,25 @@ Weights are downloaded automatically when instantiating a model. They are stored
 
 ## Available models
 
-Models for image classification with weights trained on ImageNet:
+### Models for image classification with weights trained on ImageNet:
 
+- [Xception](#xception)
 - [VGG16](#vgg16)
 - [VGG19](#vgg19)
 - [ResNet50](#resnet50)
 - [InceptionV3](#inceptionv3)
 
-All of these architectures are compatible with both TensorFlow and Theano, and upon instantiation the models will be built according to the image dimension ordering set in your Keras configuration file at `~/.keras/keras.json`. For instance, if you have set `image_dim_ordering=tf`, then any model loaded from this repository will get built according to the TensorFlow dimension ordering convention, "Width-Height-Depth".
+All of these architectures (except Xception) are compatible with both TensorFlow and Theano, and upon instantiation the models will be built according to the image dimension ordering set in your Keras configuration file at `~/.keras/keras.json`. For instance, if you have set `image_dim_ordering=tf`, then any model loaded from this repository will get built according to the TensorFlow dimension ordering convention, "Width-Height-Depth".
+
+The Xception model is only available for TensorFlow, due to its reliance on `SeparableConvolution` layers.
+
+### Model for music audio file auto-tagging (taking as input Mel-spectrograms):
+
+- [MusicTaggerCRNN](#musictaggercrnn)
 
 -----
 
-## Examples
+## Usage examples for image classification models
 
 ### Classify ImageNet classes with ResNet50
 
@@ -26,6 +33,7 @@ All of these architectures are compatible with both TensorFlow and Theano, and u
 from keras.applications.resnet50 import ResNet50
 from keras.preprocessing import image
 from keras.applications.resnet50 import preprocess_input, decode_predictions
+import numpy as np
 
 model = ResNet50(weights='imagenet')
 
@@ -36,8 +44,10 @@ x = np.expand_dims(x, axis=0)
 x = preprocess_input(x)
 
 preds = model.predict(x)
-print('Predicted:', decode_predictions(preds))
-# print: [[u'n02504458', u'African_elephant']]
+# decode the results into a list of tuples (class, description, probability)
+# (one such list for each sample in the batch)
+print('Predicted:', decode_predictions(preds, top=3)[0])
+# Predicted: [(u'n02504013', u'Indian_elephant', 0.82658225), (u'n01871265', u'tusker', 0.1122357), (u'n02504458', u'African_elephant', 0.061040461)]
 ```
 
 ### Extract features with VGG16
@@ -46,6 +56,7 @@ print('Predicted:', decode_predictions(preds))
 from keras.applications.vgg16 import VGG16
 from keras.preprocessing import image
 from keras.applications.vgg16 import preprocess_input
+import numpy as np
 
 model = VGG16(weights='imagenet', include_top=False)
 
@@ -65,6 +76,7 @@ from keras.applications.vgg19 import VGG19
 from keras.preprocessing import image
 from keras.applications.vgg19 import preprocess_input
 from keras.models import Model
+import numpy as np
 
 base_model = VGG19(weights='imagenet')
 model = Model(input=base_model.input, output=base_model.get_layer('block4_pool').output)
@@ -153,18 +165,96 @@ model = InceptionV3(input_tensor=input_tensor, weights='imagenet', include_top=T
 
 -----
 
+# Documentation for individual models
+
+- [Xception](#xception)
+- [VGG16](#vgg16)
+- [VGG19](#vgg19)
+- [ResNet50](#resnet50)
+- [InceptionV3](#inceptionv3)
+- [MusicTaggerCRNN](#musictaggercrnn)
+
+-----
+
+
+## Xception
+
+
+```python
+keras.applications.xception.Xception(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+Xception V1 model, with weights pre-trained on ImageNet.
+
+On ImageNet, this model gets to a top-1 validation accuracy of 0.790
+and a top-5 validation accuracy of 0.945.
+
+Note that this model is only available for the TensorFlow backend,
+due to its reliance on `SeparableConvolution` layers. Additionally it only supports
+the dimension ordering "tf" (width, height, channels).
+
+The default input size for this model is 299x299.
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(299, 299, 3)`.
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 71.
+    E.g. `(150, 150, 3)` would be one valid value.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Xception: Deep Learning with Depthwise Separable Convolutions](https://arxiv.org/abs/1610.02357)
+
+### License
+
+These weights are trained by ourselves and are released under the MIT license.
+
+
+-----
+
+
 ## VGG16
 
 ```python
-keras.applications.vgg16.VGG16(include_top=True, weights='imagenet', input_tensor=None)
+keras.applications.vgg16.VGG16(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
 ```
 
+VGG16 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
 ### Arguments
 
 - include_top: whether to include the 3 fully-connected layers at the top of the network.
 - weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
 - input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
-
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(224, 224, 3)` (with `tf` dim ordering)
+    or `(3, 224, 244)` (with `th` dim ordering).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 48.
+    E.g. `(200, 200, 3)` would be one valid value.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
 ### Returns
 
 A Keras model instance.
@@ -183,15 +273,33 @@ These weights are ported from the ones [released by VGG at Oxford](http://www.ro
 
 
 ```python
-keras.applications.vgg19.VGG19(include_top=True, weights='imagenet', input_tensor=None)
+keras.applications.vgg19.VGG19(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
 ```
 
+
+VGG19 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
 ### Arguments
 
 - include_top: whether to include the 3 fully-connected layers at the top of the network.
 - weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
 - input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
-
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(224, 224, 3)` (with `tf` dim ordering)
+    or `(3, 224, 244)` (with `th` dim ordering).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 48.
+    E.g. `(200, 200, 3)` would be one valid value.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
 ### Returns
 
 A Keras model instance.
@@ -211,15 +319,34 @@ These weights are ported from the ones [released by VGG at Oxford](http://www.ro
 
 
 ```python
-keras.applications.resnet50.ResNet50(include_top=True, weights='imagenet', input_tensor=None)
+keras.applications.resnet50.ResNet50(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
 ```
 
+
+ResNet50 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
+
 ### Arguments
 
-- include_top: whether to include the 3 fully-connected layers at the top of the network.
+- include_top: whether to include the fully-connected layer at the top of the network.
 - weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
 - input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
-
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(224, 224, 3)` (with `tf` dim ordering)
+    or `(3, 224, 244)` (with `th` dim ordering).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 197.
+    E.g. `(200, 200, 3)` would be one valid value.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
 ### Returns
 
 A Keras model instance.
@@ -238,15 +365,33 @@ These weights are ported from the ones [released by Kaiming He](https://github.c
 
 
 ```python
-keras.applications.inception_v3.InceptionV3(include_top=True, weights='imagenet', input_tensor=None)
+keras.applications.inception_v3.InceptionV3(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
 ```
 
+Inception V3 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 299x299.
+
+
 ### Arguments
 
-- include_top: whether to include the 3 fully-connected layers at the top of the network.
+- include_top: whether to include the fully-connected layer at the top of the network.
 - weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
 - input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
-
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(299, 299, 3)` (with `tf` dim ordering)
+    or `(3, 299, 299)` (with `th` dim ordering).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 139.
+    E.g. `(150, 150, 3)` would be one valid value.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
 ### Returns
 
 A Keras model instance.
@@ -258,3 +403,67 @@ A Keras model instance.
 ### License
 
 These weights are trained by ourselves and are released under the MIT license.
+
+-----
+
+## MusicTaggerCRNN
+
+
+```python
+keras.applications.music_tagger_crnn.MusicTaggerCRNN(weights='msd', input_tensor=None, include_top=True)
+```
+
+A convolutional-recurrent model taking as input a vectorized representation of the MelSpectrogram of a music track and capable of outputting the musical genre of the track. You can use `keras.applications.music_tagger_crnn.preprocess_input` to convert a sound file to a vectorized spectrogram. This requires to have installed the [Librosa](http://librosa.github.io/librosa/) library. See [the usage example](#music-tagging-and-feature-extraction-with-musictaggercrnn).
+
+### Arguments
+
+- weights: one of `None` (random initialization) or "msd" (pre-training on [Million Song Dataset](http://labrosa.ee.columbia.edu/millionsong/)).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- include_top: whether to include the 1 fully-connected layer (output layer) at the top of the network. If False, the network outputs 32-dim features.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Convolutional Recurrent Neural Networks for Music Classification](https://arxiv.org/abs/1609.04243)
+
+### License
+
+These weights are ported from the ones [released by Keunwoo Choi](https://github.com/keunwoochoi/music-auto_tagging-keras) under the [MIT license](https://github.com/keunwoochoi/music-auto_tagging-keras/blob/master/LICENSE.md).
+
+### Examples: music tagging and audio feature extraction
+
+```python
+from keras.applications.music_tagger_crnn import MusicTaggerCRNN
+from keras.applications.music_tagger_crnn import preprocess_input, decode_predictions
+import numpy as np
+
+# 1. Tagging
+model = MusicTaggerCRNN(weights='msd')
+
+audio_path = 'audio_file.mp3'
+melgram = preprocess_input(audio_path)
+melgrams = np.expand_dims(melgram, axis=0)
+
+preds = model.predict(melgrams)
+print('Predicted:')
+print(decode_predictions(preds))
+# print: ('Predicted:', [[('rock', 0.097071797), ('pop', 0.042456303), ('alternative', 0.032439161), ('indie', 0.024491295), ('female vocalists', 0.016455274)]])
+
+#. 2. Feature extraction
+model = MusicTaggerCRNN(weights='msd', include_top=False)
+
+audio_path = 'audio_file.mp3'
+melgram = preprocess_input(audio_path)
+melgrams = np.expand_dims(melgram, axis=0)
+
+feats = model.predict(melgrams)
+print('Features:')
+print(feats[0, :10])
+# print: ('Features:', [-0.19160545 0.94259131 -0.9991011 0.47644514 -0.19089699 0.99033844 0.1103896 -0.00340496 0.14823607 0.59856361])
+```

docs/templates/backend.md

@@ -44,6 +44,29 @@ Using TensorFlow backend.
 
 ----
 
+## keras.json details
+
+
+```
+{
+    "image_dim_ordering": "tf",
+    "epsilon": 1e-07,
+    "floatx": "float32",
+    "backend": "tensorflow"
+}
+```
+
+You can change these settings by editing `~/.keras/keras.json`. 
+
+* `image_dim_ordering`: string, either `"tf"` or `"th"`. It specifies which dimension ordering convention Keras will follow. (`keras.backend.image_dim_ordering()` returns it.)
+  - For 2D data (e.g. image), `"tf"` assumes `(rows, cols, channels)` while `"th"` assumes `(channels, rows, cols)`. 
+  - For 3D data, `"tf"` assumes `(conv_dim1, conv_dim2, conv_dim3, channels)` while `"th"` assumes `(channels, conv_dim1, conv_dim2, conv_dim3)`.
+* `epsilon`: float, a numeric fuzzing constant used to avoid dividing by zero in some operations.
+* `floatx`: string, `"float16"`, `"float32"`, or `"float64"`. Default float precision.
+* `backend`: string, `"tensorflow"` or `"theano"`.
+
+----
+
 ## Using the abstract Keras backend to write new code
 
 If you want the Keras modules you write to be compatible with both Theano and TensorFlow, you have to write them via the abstract Keras backend API. Here's an intro.

docs/templates/constraints.md

@@ -2,7 +2,7 @@
 
 Functions from the `constraints` module allow setting constraints (eg. non-negativity) on network parameters during optimization.
 
-The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `TimeDistributedDense`, `MaxoutDense`, `Convolution1D` and `Convolution2D` have a unified API.
+The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `TimeDistributedDense`, `MaxoutDense`, `Convolution1D`, `Convolution2D` and `Convolution3D` have a unified API.
 
 These layers expose 2 keyword arguments:
 

docs/templates/getting-started/faq.md

@@ -2,9 +2,10 @@
 
 - [How should I cite Keras?](#how-should-i-cite-keras)
 - [How can I run Keras on GPU?](#how-can-i-run-keras-on-gpu)
+- [What does \["sample", "batch", "epoch"\] mean?](#what-does-sample-batch-epoch-mean)
 - [How can I save a Keras model?](#how-can-i-save-a-keras-model)
 - [Why is the training loss much higher than the testing loss?](#why-is-the-training-loss-much-higher-than-the-testing-loss)
-- [How can I visualize the output of an intermediate layer?](#how-can-i-visualize-the-output-of-an-intermediate-layer)
+- [How can I obtain the output of an intermediate layer?](#how-can-i-obtain-the-output-of-an-intermediate-layer)
 - [How can I use Keras with datasets that don't fit in memory?](#how-can-i-use-keras-with-datasets-that-dont-fit-in-memory)
 - [How can I interrupt training when the validation loss isn't decreasing anymore?](#how-can-i-interrupt-training-when-the-validation-loss-isnt-decreasing-anymore)
 - [How is the validation split computed?](#how-is-the-validation-split-computed)
@@ -31,6 +32,8 @@ Please cite Keras in your publications if it helps your research. Here is an exa
 }
 ```
 
+---
+
 ### How can I run Keras on GPU?
 
 If you are running on the TensorFlow backend, your code will automatically run on GPU if any available GPU is detected.
@@ -54,6 +57,21 @@ theano.config.floatX = 'float32'
 
 ---
 
+### What does \["sample", "batch", "epoch"\] mean?
+
+Below are some common definitions that are necessary to know and understand to correctly utilize Keras:
+
+- **Sample**: one element of a dataset.
+  - *Example:* one image is a **sample** in a convolutional network
+  - *Example:* one audio file is a **sample** for a speech recognition model
+- **Batch**: a set of *N* samples. The samples in a **batch** are processed independently, in parallel. If training, a batch results in only one update to the model.
+  - A **batch** generally approximates the distribution of the input data better than a single input. The larger the batch, the better the approximation; however, it is also true that the batch will take longer to processes and will still result in only one update. For inference (evaluate/predict), it is recommended to pick a batch size that is as large as you can afford without going out of memory (since larger batches will usually result in faster evaluating/prediction).
+- **Epoch**: an arbitrary cutoff, generally defined as "one pass over the entire dataset", used to separate training into distinct phases, which is useful for logging and periodic evaluation.
+  - When using `evaluation_data` or `evaluation_split` with the `fit` method of Keras models, evaluation will be run at the end of every **epoch**.
+  - Within Keras, there is the ability to add [callbacks](https://keras.io/callbacks/) specifically designed to be run at the end of an **epoch**. Examples of these are learning rate changes and model checkpointing (saving).
+
+---
+
 ### How can I save a Keras model?
 
 *It is not recommended to use pickle or cPickle to save a Keras model.*
@@ -156,9 +174,22 @@ Besides, the training loss is the average of the losses over each batch of train
 
 ---
 
-### How can I visualize the output of an intermediate layer?
+### How can I obtain the output of an intermediate layer?
 
-You can build a Keras function that will return the output of a certain layer given a certain input, for example:
+One simple way is to create a new `Model` that will output the layers that you are interested in:
+
+```python
+from keras.models import Model
+
+model = ...  # create the original model
+
+layer_name = 'my_layer'
+intermediate_layer_model = Model(input=model.input,
+                                 output=model.get_layer(layer_name).output)
+intermediate_output = intermediate_layer_model.predict(data)
+```
+
+Alternatively, you can build a Keras function that will return the output of a certain layer given a certain input, for example:
 
 ```python
 from keras import backend as K
@@ -185,22 +216,6 @@ layer_output = get_3rd_layer_output([X, 0])[0]
 layer_output = get_3rd_layer_output([X, 1])[0]
 ```
 
-Another more flexible way of getting output from intermediate layers is to use the [functional API](/getting-started/functional-api-guide). For example, if you have created an autoencoder for MNIST:
-
-```python
-inputs = Input(shape=(784,))
-encoded = Dense(32, activation='relu')(inputs)
-decoded = Dense(784)(encoded)
-model = Model(input=inputs, output=decoded)
-```
-
-After compiling and training the model, you can get the output of the data from the encoder like this:
-
-```python
-encoder = Model(input=inputs, output=encoded)
-X_encoded = encoder.predict(X)
-```
-
 ---
 
 ### How can I use Keras with datasets that don't fit in memory?
@@ -229,8 +244,9 @@ Find out more in the [callbacks documentation](/callbacks).
 
 ### How is the validation split computed?
 
-If you set the `validation_split` argument in `model.fit` to e.g. 0.1, then the validation data used will be the *last 10%* of the data. If you set it to 0.25, it will be the last 25% of the data, etc.
+If you set the `validation_split` argument in `model.fit` to e.g. 0.1, then the validation data used will be the *last 10%* of the data. If you set it to 0.25, it will be the last 25% of the data, etc. Note that the data isn't shuffled before extracting the validation split, so the validation is literally just the *last* x% of samples in the input you passed.
 
+The same validation set is used for all epochs (within a same call to `fit`).
 
 ---
 
@@ -358,6 +374,7 @@ print(len(model.layers))  # "1"
 
 Code and pre-trained weights are available for the following image classification models:
 
+- Xception
 - VGG16
 - VGG19
 - ResNet50
@@ -366,6 +383,7 @@ Code and pre-trained weights are available for the following image classificatio
 They can be imported from the module `keras.applications`:
 
 ```python
+from keras.applications.xception import Xception
 from keras.applications.vgg16 import VGG16
 from keras.applications.vgg19 import VGG19
 from keras.applications.resnet50 import ResNet50

docs/templates/getting-started/functional-api-guide.md

@@ -102,7 +102,7 @@ lstm_out = LSTM(32)(x)
 Here we insert the auxiliary loss, allowing the LSTM and Embedding layer to be trained smoothly even though the main loss will be much higher in the model.
 
 ```python
-auxiliary_loss = Dense(1, activation='sigmoid', name='aux_output')(lstm_out)
+auxiliary_output = Dense(1, activation='sigmoid', name='aux_output')(lstm_out)
 ```
 
 At this point, we feed into the model our auxiliary input data by concatenating it with the LSTM output:
@@ -117,13 +117,13 @@ x = Dense(64, activation='relu')(x)
 x = Dense(64, activation='relu')(x)
 
 # and finally we add the main logistic regression layer
-main_loss = Dense(1, activation='sigmoid', name='main_output')(x)
+main_output = Dense(1, activation='sigmoid', name='main_output')(x)
 ```
 
 This defines a model with two inputs and two outputs:
 
 ```python
-model = Model(input=[main_input, auxiliary_input], output=[main_loss, auxiliary_loss])
+model = Model(input=[main_input, auxiliary_input], output=[main_output, auxiliary_output])
 ```
 
 We compile the model and assign a weight of 0.2 to the auxiliary loss.

docs/templates/getting-started/sequential-model-guide.md

@@ -121,7 +121,7 @@ Before training a model, you need to configure the learning process, which is do
 
 - an optimizer. This could be the string identifier of an existing optimizer (such as `rmsprop` or `adagrad`), or an instance of the `Optimizer` class. See: [optimizers](/optimizers).
 - a loss function. This is the objective that the model will try to minimize. It can be the string identifier of an existing loss function (such as `categorical_crossentropy` or `mse`), or it can be an objective function. See: [objectives](/objectives).
-- a list of metrics. For any classification problem you will want to set this to `metrics=['accuracy']`. A metric could be the string identifier of an existing metric (only `accuracy` is supported at this point), or a custom metric function.
+- a list of metrics. For any classification problem you will want to set this to `metrics=['accuracy']`. A metric could be the string identifier of an existing metric or a custom metric function.  Custom metric function should return either a single tensor value or a dict `metric_name -> metric_value`. See: [metrics](/metrics).
 
 ```python
 # for a multi-class classification problem
@@ -137,6 +137,24 @@ model.compile(optimizer='rmsprop',
 # for a mean squared error regression problem
 model.compile(optimizer='rmsprop',
               loss='mse')
+
+# for custom metrics
+import keras.backend as K
+
+def mean_pred(y_true, y_pred):
+    return K.mean(y_pred)
+
+def false_rates(y_true, y_pred):
+    false_neg = ...
+    false_pos = ...
+    return {
+        'false_neg': false_neg,
+        'false_pos': false_pos,
+    }
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy', mean_pred, false_rates])
 ```
 
 ----

docs/templates/index.md

@@ -2,7 +2,7 @@
 
 ## You have just found Keras.
 
-Keras is a minimalist, highly modular neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
+Keras is a high-level neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
 
 Use Keras if you need a deep learning library that:
 
@@ -51,9 +51,9 @@ Stacking layers is as easy as `.add()`:
 from keras.layers import Dense, Activation
 
 model.add(Dense(output_dim=64, input_dim=100))
-model.add(Activation("relu"))
+model.add(Activation('relu'))
 model.add(Dense(output_dim=10))
-model.add(Activation("softmax"))
+model.add(Activation('softmax'))
 ```
 
 Once your model looks good, configure its learning process with `.compile()`:
@@ -143,9 +143,12 @@ By default, Keras will use TensorFlow as its tensor manipulation library. [Follo
 
 ## Support
 
-You can ask questions and join the development discussion on the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+You can ask questions and join the development discussion:
 
-You can also post bug reports and feature requests in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
+- On the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+- On the [Keras Slack channel](https://kerasteam.slack.com). Use [this link](https://keras-slack-autojoin.herokuapp.com/) to request an invitation to the channel.
+
+You can also post **bug reports and feature requests** (only) in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
 
 
 ------------------

docs/templates/initializations.md

@@ -17,6 +17,7 @@ model.add(Dense(64, init='uniform'))
 - __identity__: Use with square 2D layers (`shape[0] == shape[1]`).
 - __orthogonal__: Use with square 2D layers (`shape[0] == shape[1]`).
 - __zero__
+- __one__
 - __glorot_normal__: Gaussian initialization scaled by fan_in + fan_out (Glorot 2010)
 - __glorot_uniform__
 - __he_normal__: Gaussian initialization scaled by fan_in (He et al., 2014)
@@ -47,4 +48,4 @@ def my_init(shape, name=None):
     return initializations.normal(shape, scale=0.01, name=name)
 
 model.add(Dense(64, init=my_init))
-```
+```

docs/templates/layers/about-keras-layers.md

@@ -5,11 +5,21 @@ All Keras layers have a number of methods in common:
 - `layer.get_weights()`: returns the weights of the layer as a list of Numpy arrays.
 - `layer.set_weights(weights)`: sets the weights of the layer from a list of Numpy arrays (with the same shapes as the output of `get_weights`).
 - `layer.get_config()`: returns a dictionary containing the configuration of the layer. The layer can be reinstantiated from its config via:
+
+```python
+layer = Dense(32)
+config = layer.get_config()
+reconstructed_layer = Dense.from_config(config)
+```
+
+Or:
+
 ```python
 from keras.utils.layer_utils import layer_from_config
 
 config = layer.get_config()
-layer = layer_from_config(config)
+layer = layer_from_config({'class_name': layer.__class__.__name__,
+                           'config': config})
 ```
 
 If a layer has a single node (i.e. if it isn't a shared layer), you can get its input tensor, output tensor, input shape and output shape via:

docs/templates/layers/writing-your-own-keras-layers.md

@@ -2,9 +2,9 @@
 
 For simple, stateless custom operations, you are probably better off using `layers.core.Lambda` layers. But for any custom operation that has trainable weights, you should implement your own layer.
 
-Here is the skeleton of a Keras layer. There are only three methods you need to implement:
+Here is the skeleton of a Keras layer, **as of Keras 1.2.0** (if you have an older version, please upgrade). There are only three methods you need to implement:
 
-- `build(input_shape)`: this is where you will define your weights. Trainable weights should be added to the list `self.trainable_weights`. Other attributes of note are: `self.non_trainable_weights` (list) and `self.updates` (list of update tuples (tensor, new_tensor)). For an example of how to use `non_trainable_weights` and `updates`, see the code for the `BatchNormalization` layer.
+- `build(input_shape)`: this is where you will define your weights. This method must set `self.built = True`, which can be done by calling `super([Layer], self).build()`.
 - `call(x)`: this is where the layer's logic lives. Unless you want your layer to support masking, you only have to care about the first argument passed to `call`: the input tensor.
 - `get_output_shape_for(input_shape)`: in case your layer modifies the shape of its input, you should specify here the shape transformation logic. This allows Keras to do automatic shape inference.
 
@@ -19,10 +19,11 @@ class MyLayer(Layer):
         super(MyLayer, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        input_dim = input_shape[1]
-        initial_weight_value = np.random.random((input_dim, output_dim))
-        self.W = K.variable(initial_weight_value)
-        self.trainable_weights = [self.W]
+        # Create a trainable weight variable for this layer.
+        self.W = self.add_weight(shape=(input_shape[1], self.output_dim),
+                                 initializer='uniform',
+                                 trainable=True)
+        super(MyLayer, self).build(input_shape)  # Be sure to call this somewhere!
 
     def call(self, x, mask=None):
         return K.dot(x, self.W)
@@ -31,4 +32,4 @@ class MyLayer(Layer):
         return (input_shape[0], self.output_dim)
 ```
 
-The existing Keras layers provide ample examples of how to implement almost anything. Never hesitate to read the source code!
+The existing Keras layers provide examples of how to implement almost anything. Never hesitate to read the source code!

docs/templates/metrics.md

@@ -0,0 +1,51 @@
+
+## Usage of metrics
+
+A metric is a function that is used to judge the performance of your model. Metric functions are to be supplied in the `metrics` parameter when a model is compiled.
+
+A metric function is similar to an [objective function](/objectives), except that the results from evaluating a metric are not used when training the model.
+
+You can either pass the name of an existing metric, or pass a Theano/TensorFlow symbolic function (see [Custom metrics](#custom-metrics)).
+
+#### Arguments
+  - __y_true__: True labels. Theano/TensorFlow tensor.
+  - __y_pred__: Predictions. Theano/TensorFlow tensor of the same shape as y_true.
+
+#### Returns
+  Single tensor value representing the mean of the output array across all
+  datapoints.
+
+----
+
+## Available metrics
+
+
+{{autogenerated}}
+
+----
+
+## Custom metrics
+
+Custom metrics can be defined and passed via the compilation step. The
+function would need to take `(y_true, y_pred)` as arguments and return
+either a single tensor value or a dict `metric_name -> metric_value`.
+
+```python
+# for custom metrics
+import keras.backend as K
+
+def mean_pred(y_true, y_pred):
+    return K.mean(y_pred)
+
+def false_rates(y_true, y_pred):
+    false_neg = ...
+    false_pos = ...
+    return {
+        'false_neg': false_neg,
+        'false_pos': false_pos,
+    }
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy', mean_pred, false_rates])
+```

docs/templates/objectives.md

@@ -30,3 +30,11 @@ For a few examples of such functions, check out the [objectives source](https://
 - __kullback_leibler_divergence__ / __kld__: Information gain from a predicted probability distribution Q to a true probability distribution P. Gives a measure of difference between both distributions.
 - __poisson__: Mean of `(predictions - targets * log(predictions))`
 - __cosine_proximity__: The opposite (negative) of the mean cosine proximity between predictions and targets.
+
+**Note**: when using the `categorical_crossentropy` objective, your targets should be in categorical format (e.g. if you have 10 classes, the target for each sample should be a 10-dimensional vector that is all-zeros expect for a 1 at the index corresponding to the class of the sample). In order to convert *integer targets* into *categorical targets*, you can use the Keras utility `to_categorical`:
+
+```python
+from keras.utils.np_utils import to_categorical
+
+categorical_labels = to_categorical(int_labels, nb_classes=None)
+```

docs/templates/preprocessing/image.md

@@ -24,9 +24,9 @@ keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
 Generate batches of tensor image data with real-time data augmentation. The data will be looped over (in batches) indefinitely.
 
 - __Arguments__:
-    - __featurewise_center__: Boolean. Set input mean to 0 over the dataset.
+    - __featurewise_center__: Boolean. Set input mean to 0 over the dataset, feature-wise.
     - __samplewise_center__: Boolean. Set each sample mean to 0.
-    - __featurewise_std_normalization__: Boolean. Divide inputs by std of the dataset.
+    - __featurewise_std_normalization__: Boolean. Divide inputs by std of the dataset, feature-wise.
     - __samplewise_std_normalization__: Boolean. Divide each input by its std.
     - __zca_whitening__: Boolean. Apply ZCA whitening.
     - __rotation_range__: Int. Degree range for random rotations.
@@ -43,25 +43,33 @@ Generate batches of tensor image data with real-time data augmentation. The data
             otherwise we multiply the data by the value provided (before applying
             any other transformation).
     - __dim_ordering__: One of {"th", "tf"}.
-        "tf" mode means that the images should have shape `(samples, width, height, channels)`,
-        "th" mode means that the images should have shape `(samples, channels, width, height)`.
+        "tf" mode means that the images should have shape `(samples, height, width, channels)`,
+        "th" mode means that the images should have shape `(samples, channels, height, width)`.
         It defaults to the `image_dim_ordering` value found in your
         Keras config file at `~/.keras/keras.json`.
-        If you never set it, then it will be "th".
+        If you never set it, then it will be "tf".
 
 - __Methods__:
     - __fit(X)__: Compute the internal data stats related to the data-dependent transformations, based on an array of sample data.
         Only required if featurewise_center or featurewise_std_normalization or zca_whitening.
         - __Arguments__:
-            - __X__: sample data.
+            - __X__: sample data. Should have rank 4.
+                In case of grayscale data,
+                the channels axis should have value 1, and in case
+                of RGB data, it should have value 3.
             - __augment__: Boolean (default: False). Whether to fit on randomly augmented samples.
             - __rounds__: int (default: 1). If augment, how many augmentation passes over the data to use.
+            - __seed__: int (default: None). Random seed.
     - __flow(X, y)__: Takes numpy data & label arrays, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
         - __Arguments__:
-            - __X__: data.
+            - __X__: data. Should have rank 4.
+                In case of grayscale data,
+                the channels axis should have value 1, and in case
+                of RGB data, it should have value 3.
             - __y__: labels.
             - __batch_size__: int (default: 32).
             - __shuffle__: boolean (defaut: True).
+            - __seed__: int (default: None).
             - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
             - __save_prefix__: str (default: `''`). Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
             - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "jpeg".
@@ -69,18 +77,20 @@ Generate batches of tensor image data with real-time data augmentation. The data
             The generator loops indefinitely.
     - __flow_from_directory(directory)__: Takes the path to a directory, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
         - __Arguments__:
-            - __directory__: path to the target directory. It should contain one subdirectory per class,
-                and the subdirectories should contain PNG or JPG images. See [this script](https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d) for more details.
+            - __directory__: path to the target directory. It should contain one subdirectory per class.
+                Any PNG, JPG or BMP images inside each of the subdirectories directory tree will be included in the generator.
+                See [this script](https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d) for more details.
             - __target_size__: tuple of integers, default: `(256, 256)`. The dimensions to which all images found will be resized.
             - __color_mode__: one of "grayscale", "rbg". Default: "rgb". Whether the images will be converted to have 1 or 3 color channels.
             - __classes__: optional list of class subdirectories (e.g. `['dogs', 'cats']`). Default: None. If not provided, the list of classes will be automatically inferred (and the order of the classes, which will map to the label indices, will be alphanumeric).
             - __class_mode__: one of "categorical", "binary", "sparse" or None. Default: "categorical". Determines the type of label arrays that are returned: "categorical" will be 2D one-hot encoded labels, "binary" will be 1D binary labels, "sparse" will be 1D integer labels. If None, no labels are returned (the generator will only yield batches of image data, which is useful to use `model.predict_generator()`, `model.evaluate_generator()`, etc.).
             - __batch_size__: size of the batches of data (default: 32).
             - __shuffle__: whether to shuffle the data (default: True)
-            - __seed__: optional random seed for shuffling.
+            - __seed__: optional random seed for shuffling and transformations.
             - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
             - __save_prefix__: str. Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
             - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "jpeg".
+            - __follow_links__: whether to follow symlinks inside class subdirectories (default: False).
 
 
 - __Examples__:
@@ -151,3 +161,40 @@ model.fit_generator(
         validation_data=validation_generator,
         nb_val_samples=800)
 ```
+
+Example of transforming images and masks together.
+
+```python
+# we create two instances with the same arguments
+data_gen_args = dict(featurewise_center=True,
+                     featurewise_std_normalization=True,
+                     rotation_range=90.,
+                     width_shift_range=0.1,
+                     height_shift_range=0.1,
+                     zoom_range=0.2)
+image_datagen = ImageDataGenerator(**data_gen_args)
+mask_datagen = ImageDataGenerator(**data_gen_args)
+
+# Provide the same seed and keyword arguments to the fit and flow methods
+seed = 1
+image_datagen.fit(images, augment=True, seed=seed)
+mask_datagen.fit(masks, augment=True, seed=seed)
+
+image_generator = image_datagen.flow_from_directory(
+    'data/images',
+    class_mode=None,
+    seed=seed)
+
+mask_generator = mask_datagen.flow_from_directory(
+    'data/masks',
+    class_mode=None,
+    seed=seed)
+
+# combine generators into one which yields image and masks
+train_generator = zip(image_generator, mask_generator)
+
+model.fit_generator(
+    train_generator,
+    samples_per_epoch=2000,
+    nb_epoch=50)
+```

docs/templates/preprocessing/sequence.md

@@ -1,10 +1,11 @@
 ## pad_sequences
 
 ```python
-keras.preprocessing.sequence.pad_sequences(sequences, maxlen=None, dtype='int32')
+keras.preprocessing.sequence.pad_sequences(sequences, maxlen=None, dtype='int32',
+    padding='pre', truncating='pre', value=0.)
 ```
 
-Transform a list of `nb_samples sequences` (lists of scalars) into a 2D Numpy array of shape `(nb_samples, nb_timesteps)`. `nb_timesteps` is either the `maxlen` argument if provided, or the length of the longest sequence otherwise. Sequences that are shorter than `nb_timesteps` are padded with zeros at the end.
+Transform a list of `nb_samples` sequences (lists of scalars) into a 2D Numpy array of shape `(nb_samples, nb_timesteps)`. `nb_timesteps` is either the `maxlen` argument if provided, or the length of the longest sequence otherwise. Sequences that are shorter than `nb_timesteps` are padded with `value` at the end. Sequences longer than `nb_timesteps` are truncated so that it fits the desired length. Position where padding or truncation happens is determined by `padding` or `truncating`, respectively.
 
 - __Return__: 2D Numpy array of shape `(nb_samples, nb_timesteps)`.
 

docs/templates/regularizers.md

@@ -2,7 +2,7 @@
 
 Regularizers allow to apply penalties on layer parameters or layer activity during optimization. These penalties are incorporated in the loss function that the network optimizes.
 
-The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `TimeDistributedDense`, `MaxoutDense`, `Convolution1D` and `Convolution2D` have a unified API.
+The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `TimeDistributedDense`, `MaxoutDense`, `Convolution1D`, `Convolution2D` and `Convolution3D` have a unified API.
 
 These layers expose 3 keyword arguments:
 

examples/README.md

@@ -0,0 +1,100 @@
+# Keras examples directory
+
+[addition_rnn.py](addition_rnn.py)
+Implementation of sequence to sequence learning for performing addition of two numbers (as strings).
+
+[antirectifier.py](antirectifier.py)
+Demonstrates how to write custom layers for Keras.
+
+[babi_memnn.py](babi_memnn.py)
+Trains a memory network on the bAbI dataset for reading comprehension.
+
+[babi_rnn.py](babi_rnn.py)
+Trains a two-branch recurrent network on the bAbI dataset for reading comprehension.
+
+[cifar10_cnn.py](cifar10_cnn.py)
+Trains a simple deep CNN on the CIFAR10 small images dataset.
+
+[conv_filter_visualization.py](conv_filter_visualization.py)
+Visualization of the filters of VGG16, via gradient ascent in input space.
+
+[conv_lstm.py](conv_lstm.py)
+Demonstrates the use of a convolutional LSTM network.
+
+[deep_dream.py](deep_dream.py)
+Deep Dreams in Keras.
+
+[image_ocr.py](image_ocr.py)
+Trains a convolutional stack followed by a recurrent stack and a CTC logloss function to perform optical character recognition (OCR).
+
+[imdb_bidirectional_lstm.py](imdb_bidirectional_lstm.py)
+Trains a Bidirectional LSTM on the IMDB sentiment classification task.
+
+[imdb_cnn.py](imdb_cnn.py)
+Demonstrates the use of Convolution1D for text classification.
+
+[imdb_cnn_lstm.py](imdb_cnn_lstm.py)
+Trains a convolutional stack followed by a recurrent stack network on the IMDB sentiment classification task.
+
+[imdb_fasttext.py](imdb_fasttext.py)
+Trains a FastText model on the IMDB sentiment classification task.
+
+[imdb_lstm.py](imdb_lstm.py)
+Trains a LSTM on the IMDB sentiment classification task.
+
+[lstm_benchmark.py](lstm_benchmark.py)
+Compares different LSTM implementations on the IMDB sentiment classification task.
+
+[lstm_text_generation.py](lstm_text_generation.py)
+Generates text from Nietzsche's writings.
+
+[mnist_acgan.py](mnist_acgan.py)
+Implementation of AC-GAN ( Auxiliary Classifier GAN ) on the MNIST dataset
+
+[mnist_cnn.py](mnist_cnn.py)
+Trains a simple convnet on the MNIST dataset.
+
+[mnist_hierarchical_rnn.py](mnist_hierarchical_rnn.py)
+Trains a Hierarchical RNN (HRNN) to classify MNIST digits.
+
+[mnist_irnn.py](mnist_irnn.py)
+Reproduction of the IRNN experiment with pixel-by-pixel sequential MNIST in "A Simple Way to Initialize Recurrent Networks of Rectified Linear Units" by Le et al.
+
+[mnist_mlp.py](mnist_mlp.py)
+Trains a simple deep multi-layer perceptron on the MNIST dataset.
+
+[mnist_net2net.py](mnist_net2net.py)
+Reproduction of the Net2Net experiment with MNIST in "Net2Net: Accelerating Learning via Knowledge Transfer".
+
+[mnist_siamese_graph.py](mnist_siamese_graph.py)
+Trains a Siamese multi-layer perceptron on pairs of digits from the MNIST dataset.
+
+[mnist_sklearn_wrapper.py](mnist_sklearn_wrapper.py)
+Demonstrates how to use the sklearn wrapper.
+
+[mnist_swwae.py](mnist_swwae.py)
+Trains a Stacked What-Where AutoEncoder built on residual blocks on the MNIST dataset.
+
+[mnist_transfer_cnn.py](mnist_transfer_cnn.py)
+Transfer learning toy example.
+
+[neural_doodle.py](neural_doodle.py)
+Neural doodle.
+
+[neural_style_transfer.py](neural_style_transfer.py)
+Neural style transfer.
+
+[pretrained_word_embeddings.py](pretrained_word_embeddings.py)
+Loads pre-trained word embeddings (GloVe embeddings) into a frozen Keras Embedding layer, and uses it to train a text classification model on the 20 Newsgroup dataset.
+
+[reuters_mlp.py](reuters_mlp.py)
+Trains and evaluate a simple MLP on the Reuters newswire topic classification task.
+
+[stateful_lstm.py](stateful_lstm.py)
+Demonstrates how to use stateful RNNs to model long sequences efficiently.
+
+[variational_autoencoder.py](variational_autoencoder.py)
+Demonstrates how to build a variational autoencoder.
+
+[variational_autoencoder_deconv.py](variational_autoencoder_deconv.py)
+Demonstrates how to build a variational autoencoder with Keras using deconvolution layers.

examples/addition_rnn.py

@@ -35,21 +35,29 @@ from six.moves import range
 
 
 class CharacterTable(object):
-    '''
-    Given a set of characters:
+    """Given a set of characters:
     + Encode them to a one hot integer representation
     + Decode the one hot integer representation to their character output
     + Decode a vector of probabilities to their character output
-    '''
-    def __init__(self, chars, maxlen):
+    """
+    def __init__(self, chars):
+        """Initialize character table.
+
+        # Arguments
+            chars: Characters that can appear in the input.
+        """
         self.chars = sorted(set(chars))
         self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
         self.indices_char = dict((i, c) for i, c in enumerate(self.chars))
-        self.maxlen = maxlen
 
-    def encode(self, C, maxlen=None):
-        maxlen = maxlen if maxlen else self.maxlen
-        X = np.zeros((maxlen, len(self.chars)))
+    def encode(self, C, num_rows):
+        """One hot encode given string C.
+
+        # Arguments
+            num_rows: Number of rows in the returned one hot encoding. This is
+                used to keep the # of rows for each data the same.
+        """
+        X = np.zeros((num_rows, len(self.chars)))
         for i, c in enumerate(C):
             X[i, self.char_indices[c]] = 1
         return X
@@ -65,40 +73,42 @@ class colors:
     fail = '\033[91m'
     close = '\033[0m'
 
-# Parameters for the model and dataset
+# Parameters for the model and dataset.
 TRAINING_SIZE = 50000
 DIGITS = 3
 INVERT = True
-# Try replacing GRU, or SimpleRNN
-RNN = recurrent.LSTM
-HIDDEN_SIZE = 128
-BATCH_SIZE = 128
-LAYERS = 1
+
+# Maximum length of input is 'int + int' (e.g., '345+678'). Maximum length of
+# int is DIGITS.
 MAXLEN = DIGITS + 1 + DIGITS
 
+# All the numbers, plus sign and space for padding.
 chars = '0123456789+ '
-ctable = CharacterTable(chars, MAXLEN)
+ctable = CharacterTable(chars)
 
 questions = []
 expected = []
 seen = set()
 print('Generating data...')
 while len(questions) < TRAINING_SIZE:
-    f = lambda: int(''.join(np.random.choice(list('0123456789')) for i in range(np.random.randint(1, DIGITS + 1))))
+    f = lambda: int(''.join(np.random.choice(list('0123456789'))
+                    for i in range(np.random.randint(1, DIGITS + 1))))
     a, b = f(), f()
     # Skip any addition questions we've already seen
-    # Also skip any such that X+Y == Y+X (hence the sorting)
+    # Also skip any such that X+Y == Y+X (hence the sorting).
     key = tuple(sorted((a, b)))
     if key in seen:
         continue
     seen.add(key)
-    # Pad the data with spaces such that it is always MAXLEN
+    # Pad the data with spaces such that it is always MAXLEN.
     q = '{}+{}'.format(a, b)
     query = q + ' ' * (MAXLEN - len(q))
     ans = str(a + b)
-    # Answers can be of maximum size DIGITS + 1
+    # Answers can be of maximum size DIGITS + 1.
     ans += ' ' * (DIGITS + 1 - len(ans))
     if INVERT:
+        # Reverse the query, e.g., '12+345  ' becomes '  543+21'. (Note the
+        # space used for padding.)
         query = query[::-1]
     questions.append(query)
     expected.append(ans)
@@ -108,53 +118,73 @@ print('Vectorization...')
 X = np.zeros((len(questions), MAXLEN, len(chars)), dtype=np.bool)
 y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool)
 for i, sentence in enumerate(questions):
-    X[i] = ctable.encode(sentence, maxlen=MAXLEN)
+    X[i] = ctable.encode(sentence, MAXLEN)
 for i, sentence in enumerate(expected):
-    y[i] = ctable.encode(sentence, maxlen=DIGITS + 1)
+    y[i] = ctable.encode(sentence, DIGITS + 1)
 
-# Shuffle (X, y) in unison as the later parts of X will almost all be larger digits
+# Shuffle (X, y) in unison as the later parts of X will almost all be larger
+# digits.
 indices = np.arange(len(y))
 np.random.shuffle(indices)
 X = X[indices]
 y = y[indices]
 
-# Explicitly set apart 10% for validation data that we never train over
-split_at = len(X) - len(X) / 10
+# Explicitly set apart 10% for validation data that we never train over.
+split_at = len(X) - len(X) // 10
 (X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
 (y_train, y_val) = (y[:split_at], y[split_at:])
 
+print('Training Data:')
 print(X_train.shape)
 print(y_train.shape)
 
+print('Validation Data:')
+print(X_val.shape)
+print(y_val.shape)
+
+# Try replacing GRU, or SimpleRNN.
+RNN = recurrent.LSTM
+HIDDEN_SIZE = 128
+BATCH_SIZE = 128
+LAYERS = 1
+
 print('Build model...')
 model = Sequential()
-# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
-# note: in a situation where your input sequences have a variable length,
+# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE.
+# Note: In a situation where your input sequences have a variable length,
 # use input_shape=(None, nb_feature).
 model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars))))
-# For the decoder's input, we repeat the encoded input for each time step
+# As the decoder RNN's input, repeatedly provide with the last hidden state of
+# RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum
+# length of output, e.g., when DIGITS=3, max output is 999+999=1998.
 model.add(RepeatVector(DIGITS + 1))
-# The decoder RNN could be multiple layers stacked or a single layer
+# The decoder RNN could be multiple layers stacked or a single layer.
 for _ in range(LAYERS):
+    # By setting return_sequences to True, return not only the last output but
+    # all the outputs so far in the form of (nb_samples, timesteps,
+    # output_dim). This is necessary as TimeDistributed in the below expects
+    # the first dimension to be the timesteps.
     model.add(RNN(HIDDEN_SIZE, return_sequences=True))
 
-# For each of step of the output sequence, decide which character should be chosen
+# Apply a dense layer to the every temporal slice of an input. For each of step
+# of the output sequence, decide which character should be chosen.
 model.add(TimeDistributed(Dense(len(chars))))
 model.add(Activation('softmax'))
-
 model.compile(loss='categorical_crossentropy',
               optimizer='adam',
               metrics=['accuracy'])
+model.summary()
 
-# Train the model each generation and show predictions against the validation dataset
+# Train the model each generation and show predictions against the validation
+# dataset.
 for iteration in range(1, 200):
     print()
     print('-' * 50)
     print('Iteration', iteration)
     model.fit(X_train, y_train, batch_size=BATCH_SIZE, nb_epoch=1,
               validation_data=(X_val, y_val))
-    ###
-    # Select 10 samples from the validation set at random so we can visualize errors
+    # Select 10 samples from the validation set at random so we can visualize
+    # errors.
     for i in range(10):
         ind = np.random.randint(0, len(X_val))
         rowX, rowy = X_val[np.array([ind])], y_val[np.array([ind])]
@@ -164,5 +194,9 @@ for iteration in range(1, 200):
         guess = ctable.decode(preds[0], calc_argmax=False)
         print('Q', q[::-1] if INVERT else q)
         print('T', correct)
-        print(colors.ok + '☑' + colors.close if correct == guess else colors.fail + '☒' + colors.close, guess)
+        if correct == guess:
+            print(colors.ok + '☑' + colors.close, end=" ")
+        else:
+            print(colors.fail + '☒' + colors.close, end=" ")
+        print(guess)
         print('---')

examples/antirectifier.py

@@ -45,6 +45,7 @@ class Antirectifier(Layer):
         with twice less parameters yet with comparable
         classification accuracy as an equivalent ReLU-based network.
     '''
+
     def get_output_shape_for(self, input_shape):
         shape = list(input_shape)
         assert len(shape) == 2  # only valid for 2D tensors

examples/cifar10_cnn.py

@@ -1,14 +1,10 @@
 '''Train a simple deep CNN on the CIFAR10 small images dataset.
 
-GPU run command:
+GPU run command with Theano backend (with TensorFlow, the GPU is automatically used):
     THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python cifar10_cnn.py
 
 It gets down to 0.65 test logloss in 25 epochs, and down to 0.55 after 50 epochs.
 (it's still underfitting at that point, though).
-
-Note: the data was pickled with Python 2, and some encoding issues might prevent you
-from loading it in Python 3. You might have to load it in Python 2,
-save it in a different format, load it in Python 3 and repickle it.
 '''
 
 from __future__ import print_function
@@ -17,7 +13,6 @@ from keras.preprocessing.image import ImageDataGenerator
 from keras.models import Sequential
 from keras.layers import Dense, Dropout, Activation, Flatten
 from keras.layers import Convolution2D, MaxPooling2D
-from keras.optimizers import SGD
 from keras.utils import np_utils
 
 batch_size = 32
@@ -27,16 +22,16 @@ data_augmentation = True
 
 # input image dimensions
 img_rows, img_cols = 32, 32
-# the CIFAR10 images are RGB
+# The CIFAR10 images are RGB.
 img_channels = 3
 
-# the data, shuffled and split between train and test sets
+# The data, shuffled and split between train and test sets:
 (X_train, y_train), (X_test, y_test) = cifar10.load_data()
 print('X_train shape:', X_train.shape)
 print(X_train.shape[0], 'train samples')
 print(X_test.shape[0], 'test samples')
 
-# convert class vectors to binary class matrices
+# Convert class vectors to binary class matrices.
 Y_train = np_utils.to_categorical(y_train, nb_classes)
 Y_test = np_utils.to_categorical(y_test, nb_classes)
 
@@ -64,10 +59,9 @@ model.add(Dropout(0.5))
 model.add(Dense(nb_classes))
 model.add(Activation('softmax'))
 
-# let's train the model using SGD + momentum (how original).
-sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
+# Let's train the model using RMSprop
 model.compile(loss='categorical_crossentropy',
-              optimizer=sgd,
+              optimizer='rmsprop',
               metrics=['accuracy'])
 
 X_train = X_train.astype('float32')
@@ -84,8 +78,7 @@ if not data_augmentation:
               shuffle=True)
 else:
     print('Using real-time data augmentation.')
-
-    # this will do preprocessing and realtime data augmentation
+    # This will do preprocessing and realtime data augmentation:
     datagen = ImageDataGenerator(
         featurewise_center=False,  # set input mean to 0 over the dataset
         samplewise_center=False,  # set each sample mean to 0
@@ -98,13 +91,13 @@ else:
         horizontal_flip=True,  # randomly flip images
         vertical_flip=False)  # randomly flip images
 
-    # compute quantities required for featurewise normalization
-    # (std, mean, and principal components if ZCA whitening is applied)
+    # Compute quantities required for featurewise normalization
+    # (std, mean, and principal components if ZCA whitening is applied).
     datagen.fit(X_train)
 
-    # fit the model on the batches generated by datagen.flow()
+    # Fit the model on the batches generated by datagen.flow().
     model.fit_generator(datagen.flow(X_train, Y_train,
-                        batch_size=batch_size),
+                                     batch_size=batch_size),
                         samples_per_epoch=X_train.shape[0],
                         nb_epoch=nb_epoch,
                         validation_data=(X_test, Y_test))

examples/conv_filter_visualization.py

@@ -20,6 +20,8 @@ img_height = 128
 layer_name = 'block5_conv1'
 
 # util function to convert a tensor into a valid image
+
+
 def deprocess_image(x):
     # normalize tensor: center on 0., ensure std is 0.1
     x -= x.mean()

examples/conv_lstm.py

@@ -0,0 +1,142 @@
+""" This script demonstrates the use of a convolutional LSTM network.
+This network is used to predict the next frame of an artificially
+generated movie which contains moving squares.
+"""
+from keras.models import Sequential
+from keras.layers.convolutional import Convolution3D
+from keras.layers.convolutional_recurrent import ConvLSTM2D
+from keras.layers.normalization import BatchNormalization
+import numpy as np
+import pylab as plt
+
+# We create a layer which take as input movies of shape
+# (n_frames, width, height, channels) and returns a movie
+# of identical shape.
+
+seq = Sequential()
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   input_shape=(None, 40, 40, 1),
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(Convolution3D(nb_filter=1, kernel_dim1=1, kernel_dim2=3,
+                      kernel_dim3=3, activation='sigmoid',
+                      border_mode='same', dim_ordering='tf'))
+
+seq.compile(loss='binary_crossentropy', optimizer='adadelta')
+
+
+# Artificial data generation:
+# Generate movies with 3 to 7 moving squares inside.
+# The squares are of shape 1x1 or 2x2 pixels,
+# which move linearly over time.
+# For convenience we first create movies with bigger width and height (80x80)
+# and at the end we select a 40x40 window.
+
+def generate_movies(n_samples=1200, n_frames=15):
+    row = 80
+    col = 80
+    noisy_movies = np.zeros((n_samples, n_frames, row, col, 1), dtype=np.float)
+    shifted_movies = np.zeros((n_samples, n_frames, row, col, 1),
+                              dtype=np.float)
+
+    for i in range(n_samples):
+        # Add 3 to 7 moving squares
+        n = np.random.randint(3, 8)
+
+        for j in range(n):
+            # Initial position
+            xstart = np.random.randint(20, 60)
+            ystart = np.random.randint(20, 60)
+            # Direction of motion
+            directionx = np.random.randint(0, 3) - 1
+            directiony = np.random.randint(0, 3) - 1
+
+            # Size of the square
+            w = np.random.randint(2, 4)
+
+            for t in range(n_frames):
+                x_shift = xstart + directionx * t
+                y_shift = ystart + directiony * t
+                noisy_movies[i, t, x_shift - w: x_shift + w,
+                             y_shift - w: y_shift + w, 0] += 1
+
+                # Make it more robust by adding noise.
+                # The idea is that if during inference,
+                # the value of the pixel is not exactly one,
+                # we need to train the network to be robust and still
+                # consider it as a pixel belonging to a square.
+                if np.random.randint(0, 2):
+                    noise_f = (-1)**np.random.randint(0, 2)
+                    noisy_movies[i, t,
+                                 x_shift - w - 1: x_shift + w + 1,
+                                 y_shift - w - 1: y_shift + w + 1,
+                                 0] += noise_f * 0.1
+
+                # Shift the ground truth by 1
+                x_shift = xstart + directionx * (t + 1)
+                y_shift = ystart + directiony * (t + 1)
+                shifted_movies[i, t, x_shift - w: x_shift + w,
+                               y_shift - w: y_shift + w, 0] += 1
+
+    # Cut to a 40x40 window
+    noisy_movies = noisy_movies[::, ::, 20:60, 20:60, ::]
+    shifted_movies = shifted_movies[::, ::, 20:60, 20:60, ::]
+    noisy_movies[noisy_movies >= 1] = 1
+    shifted_movies[shifted_movies >= 1] = 1
+    return noisy_movies, shifted_movies
+
+# Train the network
+noisy_movies, shifted_movies = generate_movies(n_samples=1200)
+seq.fit(noisy_movies[:1000], shifted_movies[:1000], batch_size=10,
+        nb_epoch=300, validation_split=0.05)
+
+# Testing the network on one movie
+# feed it with the first 7 positions and then
+# predict the new positions
+which = 1004
+track = noisy_movies[which][:7, ::, ::, ::]
+
+for j in range(16):
+    new_pos = seq.predict(track[np.newaxis, ::, ::, ::, ::])
+    new = new_pos[::, -1, ::, ::, ::]
+    track = np.concatenate((track, new), axis=0)
+
+
+# And then compare the predictions
+# to the ground truth
+track2 = noisy_movies[which][::, ::, ::, ::]
+for i in range(15):
+    fig = plt.figure(figsize=(10, 5))
+
+    ax = fig.add_subplot(121)
+
+    if i >= 7:
+        ax.text(1, 3, 'Predictions !', fontsize=20, color='w')
+    else:
+        ax.text(1, 3, 'Inital trajectory', fontsize=20)
+
+    toplot = track[i, ::, ::, 0]
+
+    plt.imshow(toplot)
+    ax = fig.add_subplot(122)
+    plt.text(1, 3, 'Ground truth', fontsize=20)
+
+    toplot = track2[i, ::, ::, 0]
+    if i >= 2:
+        toplot = shifted_movies[which][i - 1, ::, ::, 0]
+
+    plt.imshow(toplot)
+    plt.savefig('%i_animate.png' % (i + 1))

examples/deep_dream.py

@@ -15,17 +15,16 @@ If running on CPU, prefer the TensorFlow backend (much faster).
 Example results: http://i.imgur.com/FX6ROg9.jpg
 '''
 from __future__ import print_function
-from scipy.misc import imread, imresize, imsave
+from keras.preprocessing.image import load_img, img_to_array
 import numpy as np
+from scipy.misc import imsave
 from scipy.optimize import fmin_l_bfgs_b
 import time
 import argparse
-import h5py
-import os
 
-from keras.models import Sequential
-from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
+from keras.applications import vgg16
 from keras import backend as K
+from keras.layers import Input
 
 parser = argparse.ArgumentParser(description='Deep Dreams with Keras.')
 parser.add_argument('base_image_path', metavar='base', type=str,
@@ -38,22 +37,22 @@ base_image_path = args.base_image_path
 result_prefix = args.result_prefix
 
 # dimensions of the generated picture.
-img_width = 600
 img_height = 600
+img_width = 600
 
 # path to the model weights file.
 weights_path = 'vgg16_weights.h5'
 
 # some settings we found interesting
 saved_settings = {
-    'bad_trip': {'features': {'conv4_1': 0.05,
-                              'conv4_2': 0.01,
-                              'conv4_3': 0.01},
+    'bad_trip': {'features': {'block4_conv1': 0.05,
+                              'block4_conv2': 0.01,
+                              'block4_conv3': 0.01},
                  'continuity': 0.1,
                  'dream_l2': 0.8,
                  'jitter': 5},
-    'dreamy': {'features': {'conv5_1': 0.05,
-                            'conv5_2': 0.02},
+    'dreamy': {'features': {'block5_conv1': 0.05,
+                            'block5_conv2': 0.02},
                'continuity': 0.1,
                'dream_l2': 0.02,
                'jitter': 0},
@@ -62,84 +61,64 @@ saved_settings = {
 settings = saved_settings['dreamy']
 
 # util function to open, resize and format pictures into appropriate tensors
+
+
 def preprocess_image(image_path):
-    img = imresize(imread(image_path), (img_width, img_height))
-    img = img.transpose((2, 0, 1)).astype('float64')
+    img = load_img(image_path, target_size=(img_height, img_width))
+    img = img_to_array(img)
     img = np.expand_dims(img, axis=0)
+    img = vgg16.preprocess_input(img)
     return img
 
 # util function to convert a tensor into a valid image
+
+
 def deprocess_image(x):
-    x = x.transpose((1, 2, 0))
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((3, img_height, img_width))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((img_height, img_width, 3))
+    # Remove zero-center by mean pixel
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
-# build the VGG16 network
-model = Sequential()
-model.add(ZeroPadding2D((1, 1), batch_input_shape=(1, 3, img_width, img_height)))
-first_layer = model.layers[-1]
-# this is a placeholder tensor that will contain our generated images
-dream = first_layer.input
+if K.image_dim_ordering() == 'th':
+    img_size = (3, img_height, img_width)
+else:
+    img_size = (img_height, img_width, 3)
+# this will contain our generated image
+dream = Input(batch_shape=(1,) + img_size)
 
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-# load the weights of the VGG16 networks
-# (trained on ImageNet, won the ILSVRC competition in 2014)
-# note: when there is a complete match between your model definition
-# and your weight savefile, you can simply call model.load_weights(filename)
-assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
-f = h5py.File(weights_path)
-for k in range(f.attrs['nb_layers']):
-    if k >= len(model.layers):
-        # we don't look at the last (fully-connected) layers in the savefile
-        break
-    g = f['layer_{}'.format(k)]
-    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
-    model.layers[k].set_weights(weights)
-f.close()
+# build the VGG16 network with our placeholder
+# the model will be loaded with pre-trained ImageNet weights
+model = vgg16.VGG16(input_tensor=dream,
+                    weights='imagenet', include_top=False)
 print('Model loaded.')
 
 # get the symbolic outputs of each "key" layer (we gave them unique names).
 layer_dict = dict([(layer.name, layer) for layer in model.layers])
 
 # continuity loss util function
+
+
 def continuity_loss(x):
     assert K.ndim(x) == 4
-    a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1])
-    b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:])
+    if K.image_dim_ordering() == 'th':
+        a = K.square(x[:, :, :img_height - 1, :img_width - 1] -
+                     x[:, :, 1:, :img_width - 1])
+        b = K.square(x[:, :, :img_height - 1, :img_width - 1] -
+                     x[:, :, :img_height - 1, 1:])
+    else:
+        a = K.square(x[:, :img_height - 1, :img_width - 1, :] -
+                     x[:, 1:, :img_width - 1, :])
+        b = K.square(x[:, :img_height - 1, :img_width - 1, :] -
+                     x[:, :img_height - 1, 1:, :])
     return K.sum(K.pow(a + b, 1.25))
 
 # define the loss
@@ -151,12 +130,15 @@ for layer_name in settings['features']:
     x = layer_dict[layer_name].output
     shape = layer_dict[layer_name].output_shape
     # we avoid border artifacts by only involving non-border pixels in the loss
-    loss -= coeff * K.sum(K.square(x[:, :, 2: shape[2]-2, 2: shape[3]-2])) / np.prod(shape[1:])
+    if K.image_dim_ordering() == 'th':
+        loss -= coeff * K.sum(K.square(x[:, :, 2: shape[2] - 2, 2: shape[3] - 2])) / np.prod(shape[1:])
+    else:
+        loss -= coeff * K.sum(K.square(x[:, 2: shape[1] - 2, 2: shape[2] - 2, :])) / np.prod(shape[1:])
 
 # add continuity loss (gives image local coherence, can result in an artful blur)
-loss += settings['continuity'] * continuity_loss(dream) / (3 * img_width * img_height)
+loss += settings['continuity'] * continuity_loss(dream) / np.prod(img_size)
 # add image L2 norm to loss (prevents pixels from taking very high values, makes image darker)
-loss += settings['dream_l2'] * K.sum(K.square(dream)) / (3 * img_width * img_height)
+loss += settings['dream_l2'] * K.sum(K.square(dream)) / np.prod(img_size)
 
 # feel free to further modify the loss as you see fit, to achieve new effects...
 
@@ -164,14 +146,16 @@ loss += settings['dream_l2'] * K.sum(K.square(dream)) / (3 * img_width * img_hei
 grads = K.gradients(loss, dream)
 
 outputs = [loss]
-if type(grads) in {list, tuple}:
+if isinstance(grads, (list, tuple)):
     outputs += grads
 else:
     outputs.append(grads)
 
 f_outputs = K.function([dream], outputs)
+
+
 def eval_loss_and_grads(x):
-    x = x.reshape((1, 3, img_width, img_height))
+    x = x.reshape((1,) + img_size)
     outs = f_outputs([x])
     loss_value = outs[0]
     if len(outs[1:]) == 1:
@@ -186,7 +170,10 @@ def eval_loss_and_grads(x):
 # "loss" and "grads". This is done because scipy.optimize
 # requires separate functions for loss and gradients,
 # but computing them separately would be inefficient.
+
+
 class Evaluator(object):
+
     def __init__(self):
         self.loss_value = None
         self.grad_values = None
@@ -215,7 +202,7 @@ for i in range(5):
     start_time = time.time()
 
     # add a random jitter to the initial image. This will be reverted at decoding time
-    random_jitter = (settings['jitter'] * 2) * (np.random.random((3, img_width, img_height)) - 0.5)
+    random_jitter = (settings['jitter'] * 2) * (np.random.random(img_size) - 0.5)
     x += random_jitter
 
     # run L-BFGS for 7 steps
@@ -223,9 +210,9 @@ for i in range(5):
                                      fprime=evaluator.grads, maxfun=7)
     print('Current loss value:', min_val)
     # decode the dream and save it
-    x = x.reshape((3, img_width, img_height))
+    x = x.reshape(img_size)
     x -= random_jitter
-    img = deprocess_image(x)
+    img = deprocess_image(np.copy(x))
     fname = result_prefix + '_at_iteration_%d.png' % i
     imsave(fname, img)
     end_time = time.time()

examples/image_ocr.py

@@ -6,36 +6,31 @@ the different fonts thrown at it...the purpose is more to demonstrate CTC
 inside of Keras.  Note that the font list may need to be updated
 for the particular OS in use.
 
-This starts off with 4 letter words. After 10 or so epochs, CTC
-learns translational invariance, so longer words and groups of words
-with spaces are gradually fed in.  This gradual increase in difficulty
-is handled using the TextImageGenerator class which is both a generator
-class for test/train data and a Keras callback class. Every 10 epochs
-the wordlist that the generator draws from increases in difficulty.
+This starts off with 4 letter words.  For the first 12 epochs, the
+difficulty is gradually increased using the TextImageGenerator class
+which is both a generator class for test/train data and a Keras
+callback class. After 20 epochs, longer sequences are thrown at it
+by recompiling the model to handle a wider image and rebuilding
+the word list to include two words separated by a space.
 
 The table below shows normalized edit distance values. Theano uses
-a slightly different CTC implementation, so some Theano-specific
-hyperparameter tuning would be needed to get it to match Tensorflow.
+a slightly different CTC implementation, hence the different results.
 
             Norm. ED
 Epoch |   TF   |   TH
 ------------------------
-    10   0.072    0.272
-    20   0.032    0.115
-    30   0.024    0.098
-    40   0.023    0.108
+    10   0.027   0.064
+    15   0.038   0.035
+    20   0.043   0.045
+    25   0.014   0.019
 
 This requires cairo and editdistance packages:
 pip install cairocffi
 pip install editdistance
 
-Due to the use of a dummy loss function, Theano requires the following flags:
-on_unused_input='ignore'
-
 Created by Mike Henry
 https://github.com/mbhenry/
 '''
-
 import os
 import itertools
 import re
@@ -47,17 +42,17 @@ from scipy import ndimage
 import pylab
 from keras import backend as K
 from keras.layers.convolutional import Convolution2D, MaxPooling2D
-from keras.layers import Input, Layer, Dense, Activation, Flatten
-from keras.layers import Reshape, Lambda, merge, Permute, TimeDistributed
+from keras.layers import Input, Dense, Activation
+from keras.layers import Reshape, Lambda, merge
 from keras.models import Model
 from keras.layers.recurrent import GRU
 from keras.optimizers import SGD
-from keras.utils import np_utils
 from keras.utils.data_utils import get_file
 from keras.preprocessing import image
 import keras.callbacks
 
-OUTPUT_DIR = "image_ocr"
+
+OUTPUT_DIR = 'image_ocr'
 
 np.random.seed(55)
 
@@ -79,28 +74,33 @@ def speckle(img):
 # also uses a random font, a slight random rotation,
 # and a random amount of speckle noise
 
-def paint_text(text, w, h):
+def paint_text(text, w, h, rotate=False, ud=False, multi_fonts=False):
     surface = cairo.ImageSurface(cairo.FORMAT_RGB24, w, h)
     with cairo.Context(surface) as context:
         context.set_source_rgb(1, 1, 1)  # White
         context.paint()
         # this font list works in Centos 7
-        fonts = ['Century Schoolbook', 'Courier', 'STIX', 'URW Chancery L', 'FreeMono']
-        context.select_font_face(np.random.choice(fonts), cairo.FONT_SLANT_NORMAL,
-                                 np.random.choice([cairo.FONT_WEIGHT_BOLD, cairo.FONT_WEIGHT_NORMAL]))
-        context.set_font_size(40)
+        if multi_fonts:
+            fonts = ['Century Schoolbook', 'Courier', 'STIX', 'URW Chancery L', 'FreeMono']
+            context.select_font_face(np.random.choice(fonts), cairo.FONT_SLANT_NORMAL,
+                                     np.random.choice([cairo.FONT_WEIGHT_BOLD, cairo.FONT_WEIGHT_NORMAL]))
+        else:
+            context.select_font_face('Courier', cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD)
+        context.set_font_size(25)
         box = context.text_extents(text)
-        if box[2] > w or box[3] > h:
+        border_w_h = (4, 4)
+        if box[2] > (w - 2 * border_w_h[1]) or box[3] > (h - 2 * border_w_h[0]):
             raise IOError('Could not fit string into image. Max char count is too large for given image width.')
 
         # teach the RNN translational invariance by
         # fitting text box randomly on canvas, with some room to rotate
-        border_w_h = (10, 16)
         max_shift_x = w - box[2] - border_w_h[0]
         max_shift_y = h - box[3] - border_w_h[1]
         top_left_x = np.random.randint(0, int(max_shift_x))
-        top_left_y = np.random.randint(0, int(max_shift_y))
-
+        if ud:
+            top_left_y = np.random.randint(0, int(max_shift_y))
+        else:
+            top_left_y = h // 2
         context.move_to(top_left_x - int(box[0]), top_left_y - int(box[1]))
         context.set_source_rgb(0, 0, 0)
         context.show_text(text)
@@ -109,10 +109,11 @@ def paint_text(text, w, h):
     a = np.frombuffer(buf, np.uint8)
     a.shape = (h, w, 4)
     a = a[:, :, 0]  # grab single channel
-    a /= 255
+    a = a.astype(np.float32) / 255
     a = np.expand_dims(a, 0)
+    if rotate:
+        a = image.random_rotation(a, 3 * (w - top_left_x) / w + 1)
     a = speckle(a)
-    a = image.random_rotation(a, 3 * (w - top_left_x) / w + 1)
 
     return a
 
@@ -164,7 +165,7 @@ def is_valid_str(in_str):
 class TextImageGenerator(keras.callbacks.Callback):
 
     def __init__(self, monogram_file, bigram_file, minibatch_size,
-                 img_w, img_h, downsample_width, val_split,
+                 img_w, img_h, downsample_factor, val_split,
                  absolute_max_string_len=16):
 
         self.minibatch_size = minibatch_size
@@ -172,7 +173,7 @@ class TextImageGenerator(keras.callbacks.Callback):
         self.img_h = img_h
         self.monogram_file = monogram_file
         self.bigram_file = bigram_file
-        self.downsample_width = downsample_width
+        self.downsample_factor = downsample_factor
         self.val_split = val_split
         self.blank_label = self.get_output_size() - 1
         self.absolute_max_string_len = absolute_max_string_len
@@ -187,7 +188,8 @@ class TextImageGenerator(keras.callbacks.Callback):
         assert num_words % self.minibatch_size == 0
         assert (self.val_split * num_words) % self.minibatch_size == 0
         self.num_words = num_words
-        self.string_list = []
+        self.string_list = [''] * self.num_words
+        tmp_string_list = []
         self.max_string_len = max_string_len
         self.Y_data = np.ones([self.num_words, self.absolute_max_string_len]) * -1
         self.X_text = []
@@ -196,25 +198,28 @@ class TextImageGenerator(keras.callbacks.Callback):
         # monogram file is sorted by frequency in english speech
         with open(self.monogram_file, 'rt') as f:
             for line in f:
-                if len(self.string_list) == int(self.num_words * mono_fraction):
+                if len(tmp_string_list) == int(self.num_words * mono_fraction):
                     break
                 word = line.rstrip()
                 if max_string_len == -1 or max_string_len is None or len(word) <= max_string_len:
-                    self.string_list.append(word)
+                    tmp_string_list.append(word)
 
         # bigram file contains common word pairings in english speech
         with open(self.bigram_file, 'rt') as f:
             lines = f.readlines()
             for line in lines:
-                if len(self.string_list) == self.num_words:
+                if len(tmp_string_list) == self.num_words:
                     break
                 columns = line.lower().split()
                 word = columns[0] + ' ' + columns[1]
                 if is_valid_str(word) and \
                         (max_string_len == -1 or max_string_len is None or len(word) <= max_string_len):
-                    self.string_list.append(word)
-        if len(self.string_list) != self.num_words:
+                    tmp_string_list.append(word)
+        if len(tmp_string_list) != self.num_words:
             raise IOError('Could not pull enough words from supplied monogram and bigram files. ')
+        # interlace to mix up the easy and hard words
+        self.string_list[::2] = tmp_string_list[:self.num_words // 2]
+        self.string_list[1::2] = tmp_string_list[self.num_words // 2:]
 
         for i, word in enumerate(self.string_list):
             self.Y_len[i] = len(word)
@@ -228,37 +233,38 @@ class TextImageGenerator(keras.callbacks.Callback):
     # each time an image is requested from train/val/test, a new random
     # painting of the text is performed
     def get_batch(self, index, size, train):
+        # width and height are backwards from typical Keras convention
+        # because width is the time dimension when it gets fed into the RNN
         if K.image_dim_ordering() == 'th':
-            X_data = np.ones([size, 1, self.img_h, self.img_w])
+            X_data = np.ones([size, 1, self.img_w, self.img_h])
         else:
-            X_data = np.ones([size, self.img_h, self.img_w, 1])
+            X_data = np.ones([size, self.img_w, self.img_h, 1])
+
         labels = np.ones([size, self.absolute_max_string_len])
         input_length = np.zeros([size, 1])
         label_length = np.zeros([size, 1])
         source_str = []
-
         for i in range(0, size):
             # Mix in some blank inputs.  This seems to be important for
             # achieving translational invariance
             if train and i > size - 4:
                 if K.image_dim_ordering() == 'th':
-                    X_data[i, 0, :, :] = paint_text('', self.img_w, self.img_h)
+                    X_data[i, 0, 0:self.img_w, :] = self.paint_func('')[0, :, :].T
                 else:
-                    X_data[i, :, :, 0] = paint_text('', self.img_w, self.img_h)
+                    X_data[i, 0:self.img_w, :, 0] = self.paint_func('',)[0, :, :].T
                 labels[i, 0] = self.blank_label
-                input_length[i] = self.downsample_width
+                input_length[i] = self.img_w // self.downsample_factor - 2
                 label_length[i] = 1
                 source_str.append('')
             else:
                 if K.image_dim_ordering() == 'th':
-                    X_data[i, 0, :, :] = paint_text(self.X_text[index + i], self.img_w, self.img_h)
+                    X_data[i, 0, 0:self.img_w, :] = self.paint_func(self.X_text[index + i])[0, :, :].T
                 else:
-                    X_data[i, :, :, 0] = paint_text(self.X_text[index + i], self.img_w, self.img_h)
+                    X_data[i, 0:self.img_w, :, 0] = self.paint_func(self.X_text[index + i])[0, :, :].T
                 labels[i, :] = self.Y_data[index + i]
-                input_length[i] = self.downsample_width
+                input_length[i] = self.img_w // self.downsample_factor - 2
                 label_length[i] = self.Y_len[index + i]
                 source_str.append(self.X_text[index + i])
-
         inputs = {'the_input': X_data,
                   'the_labels': labels,
                   'input_length': input_length,
@@ -287,19 +293,23 @@ class TextImageGenerator(keras.callbacks.Callback):
             yield ret
 
     def on_train_begin(self, logs={}):
-        # translational invariance seems to be the hardest thing
-        # for the RNN to learn, so start with <= 4 letter words.
         self.build_word_list(16000, 4, 1)
+        self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                  rotate=False, ud=False, multi_fonts=False)
 
     def on_epoch_begin(self, epoch, logs={}):
-        # After 10 epochs, translational invariance should be learned
-        # so start feeding longer words and eventually multiple words with spaces
-        if epoch == 10:
-            self.build_word_list(32000, 8, 1)
-        if epoch == 20:
-            self.build_word_list(32000, 8, 0.6)
-        if epoch == 30:
-            self.build_word_list(64000, 12, 0.5)
+        # rebind the paint function to implement curriculum learning
+        if epoch >= 3 and epoch < 6:
+            self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                      rotate=False, ud=True, multi_fonts=False)
+        elif epoch >= 6 and epoch < 9:
+            self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                      rotate=False, ud=True, multi_fonts=True)
+        elif epoch >= 9:
+            self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                      rotate=True, ud=True, multi_fonts=True)
+        if epoch >= 21 and self.max_string_len < 12:
+            self.build_word_list(32000, 12, 0.5)
 
 
 # the actual loss calc occurs here despite it not being
@@ -335,13 +345,14 @@ def decode_batch(test_func, word_batch):
 
 class VizCallback(keras.callbacks.Callback):
 
-    def __init__(self, test_func, text_img_gen, num_display_words=6):
+    def __init__(self, run_name, test_func, text_img_gen, num_display_words=6):
         self.test_func = test_func
         self.output_dir = os.path.join(
-            OUTPUT_DIR, datetime.datetime.now().strftime('%A, %d. %B %Y %I.%M%p'))
+            OUTPUT_DIR, run_name)
         self.text_img_gen = text_img_gen
         self.num_display_words = num_display_words
-        os.makedirs(self.output_dir)
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
 
     def show_edit_distance(self, num):
         num_left = num
@@ -362,109 +373,116 @@ class VizCallback(keras.callbacks.Callback):
               % (num, mean_ed, mean_norm_ed))
 
     def on_epoch_end(self, epoch, logs={}):
-        self.model.save_weights(os.path.join(self.output_dir, 'weights%02d.h5' % epoch))
+        self.model.save_weights(os.path.join(self.output_dir, 'weights%02d.h5' % (epoch)))
         self.show_edit_distance(256)
         word_batch = next(self.text_img_gen)[0]
         res = decode_batch(self.test_func, word_batch['the_input'][0:self.num_display_words])
-
+        if word_batch['the_input'][0].shape[0] < 256:
+            cols = 2
+        else:
+            cols = 1
         for i in range(self.num_display_words):
-            pylab.subplot(self.num_display_words, 1, i + 1)
+            pylab.subplot(self.num_display_words // cols, cols, i + 1)
             if K.image_dim_ordering() == 'th':
                 the_input = word_batch['the_input'][i, 0, :, :]
             else:
                 the_input = word_batch['the_input'][i, :, :, 0]
-            pylab.imshow(the_input, cmap='Greys_r')
-            pylab.xlabel('Truth = \'%s\' Decoded = \'%s\'' % (word_batch['source_str'][i], res[i]))
+            pylab.imshow(the_input.T, cmap='Greys_r')
+            pylab.xlabel('Truth = \'%s\'\nDecoded = \'%s\'' % (word_batch['source_str'][i], res[i]))
         fig = pylab.gcf()
-        fig.set_size_inches(10, 12)
-        pylab.savefig(os.path.join(self.output_dir, 'e%02d.png' % epoch))
+        fig.set_size_inches(10, 13)
+        pylab.savefig(os.path.join(self.output_dir, 'e%02d.png' % (epoch)))
         pylab.close()
 
-# Input Parameters
-img_h = 64
-img_w = 512
-nb_epoch = 50
-minibatch_size = 32
-words_per_epoch = 16000
-val_split = 0.2
-val_words = int(words_per_epoch * (val_split))
 
-# Network parameters
-conv_num_filters = 16
-filter_size = 3
-pool_size_1 = 4
-pool_size_2 = 2
-time_dense_size = 32
-rnn_size = 512
-time_steps = img_w / (pool_size_1 * pool_size_2)
+def train(run_name, start_epoch, stop_epoch, img_w):
+    # Input Parameters
+    img_h = 64
+    words_per_epoch = 16000
+    val_split = 0.2
+    val_words = int(words_per_epoch * (val_split))
 
-if K.image_dim_ordering() == 'th':
-    input_shape = (1, img_h, img_w)
-else:
-    input_shape = (img_h, img_w, 1)
+    # Network parameters
+    conv_num_filters = 16
+    filter_size = 3
+    pool_size = 2
+    time_dense_size = 32
+    rnn_size = 512
 
-fdir = os.path.dirname(get_file('wordlists.tgz',
-                                origin='http://www.isosemi.com/datasets/wordlists.tgz', untar=True))
+    if K.image_dim_ordering() == 'th':
+        input_shape = (1, img_w, img_h)
+    else:
+        input_shape = (img_w, img_h, 1)
 
-img_gen = TextImageGenerator(monogram_file=os.path.join(fdir, 'wordlist_mono_clean.txt'),
-                             bigram_file=os.path.join(fdir, 'wordlist_bi_clean.txt'),
-                             minibatch_size=32,
-                             img_w=img_w,
-                             img_h=img_h,
-                             downsample_width=img_w / (pool_size_1 * pool_size_2) - 2,
-                             val_split=words_per_epoch - val_words)
+    fdir = os.path.dirname(get_file('wordlists.tgz',
+                                    origin='http://www.isosemi.com/datasets/wordlists.tgz', untar=True))
 
-act = 'relu'
-input_data = Input(name='the_input', shape=input_shape, dtype='float32')
-inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same',
-                      activation=act, name='conv1')(input_data)
-inner = MaxPooling2D(pool_size=(pool_size_1, pool_size_1), name='max1')(inner)
-inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same',
-                      activation=act, name='conv2')(inner)
-inner = MaxPooling2D(pool_size=(pool_size_2, pool_size_2), name='max2')(inner)
+    img_gen = TextImageGenerator(monogram_file=os.path.join(fdir, 'wordlist_mono_clean.txt'),
+                                 bigram_file=os.path.join(fdir, 'wordlist_bi_clean.txt'),
+                                 minibatch_size=32,
+                                 img_w=img_w,
+                                 img_h=img_h,
+                                 downsample_factor=(pool_size ** 2),
+                                 val_split=words_per_epoch - val_words
+                                 )
+    act = 'relu'
+    input_data = Input(name='the_input', shape=input_shape, dtype='float32')
+    inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same',
+                          activation=act, init='he_normal', name='conv1')(input_data)
+    inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max1')(inner)
+    inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same',
+                          activation=act, init='he_normal', name='conv2')(inner)
+    inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max2')(inner)
 
-conv_to_rnn_dims = ((img_h / (pool_size_1 * pool_size_2)) * conv_num_filters, img_w / (pool_size_1 * pool_size_2))
-inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
-inner = Permute(dims=(2, 1), name='permute')(inner)
+    conv_to_rnn_dims = (img_w // (pool_size ** 2), (img_h // (pool_size ** 2)) * conv_num_filters)
+    inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
 
-# cuts down input size going into RNN:
-inner = TimeDistributed(Dense(time_dense_size, activation=act, name='dense1'))(inner)
+    # cuts down input size going into RNN:
+    inner = Dense(time_dense_size, activation=act, name='dense1')(inner)
 
-# Two layers of bidirecitonal GRUs
-# GRU seems to work as well, if not better than LSTM:
-gru_1 = GRU(rnn_size, return_sequences=True, name='gru1')(inner)
-gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, name='gru1_b')(inner)
-gru1_merged = merge([gru_1, gru_1b], mode='sum')
-gru_2 = GRU(rnn_size, return_sequences=True, name='gru2')(gru1_merged)
-gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True)(gru1_merged)
+    # Two layers of bidirecitonal GRUs
+    # GRU seems to work as well, if not better than LSTM:
+    gru_1 = GRU(rnn_size, return_sequences=True, init='he_normal', name='gru1')(inner)
+    gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, init='he_normal', name='gru1_b')(inner)
+    gru1_merged = merge([gru_1, gru_1b], mode='sum')
+    gru_2 = GRU(rnn_size, return_sequences=True, init='he_normal', name='gru2')(gru1_merged)
+    gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True, init='he_normal', name='gru2_b')(gru1_merged)
 
-# transforms RNN output to character activations:
-inner = TimeDistributed(Dense(img_gen.get_output_size(), name='dense2'))(merge([gru_2, gru_2b], mode='concat'))
-y_pred = Activation('softmax', name='softmax')(inner)
-Model(input=[input_data], output=y_pred).summary()
+    # transforms RNN output to character activations:
+    inner = Dense(img_gen.get_output_size(), init='he_normal',
+                  name='dense2')(merge([gru_2, gru_2b], mode='concat'))
+    y_pred = Activation('softmax', name='softmax')(inner)
+    Model(input=[input_data], output=y_pred).summary()
 
-labels = Input(name='the_labels', shape=[img_gen.absolute_max_string_len], dtype='float32')
-input_length = Input(name='input_length', shape=[1], dtype='int64')
-label_length = Input(name='label_length', shape=[1], dtype='int64')
-# Keras doesn't currently support loss funcs with extra parameters
-# so CTC loss is implemented in a lambda layer
-loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name="ctc")([y_pred, labels, input_length, label_length])
+    labels = Input(name='the_labels', shape=[img_gen.absolute_max_string_len], dtype='float32')
+    input_length = Input(name='input_length', shape=[1], dtype='int64')
+    label_length = Input(name='label_length', shape=[1], dtype='int64')
+    # Keras doesn't currently support loss funcs with extra parameters
+    # so CTC loss is implemented in a lambda layer
+    loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([y_pred, labels, input_length, label_length])
 
-lr = 0.03
-# clipnorm seems to speeds up convergence
-clipnorm = 5
-sgd = SGD(lr=lr, decay=3e-7, momentum=0.9, nesterov=True, clipnorm=clipnorm)
+    # clipnorm seems to speeds up convergence
+    sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
 
-model = Model(input=[input_data, labels, input_length, label_length], output=[loss_out])
+    model = Model(input=[input_data, labels, input_length, label_length], output=[loss_out])
 
-# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
-model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
+    # the loss calc occurs elsewhere, so use a dummy lambda func for the loss
+    model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
+    if start_epoch > 0:
+        weight_file = os.path.join(OUTPUT_DIR, os.path.join(run_name, 'weights%02d.h5' % (start_epoch - 1)))
+        model.load_weights(weight_file)
+    # captures output of softmax so we can decode the output during visualization
+    test_func = K.function([input_data], [y_pred])
 
-# captures output of softmax so we can decode the output during visualization
-test_func = K.function([input_data], [y_pred])
+    viz_cb = VizCallback(run_name, test_func, img_gen.next_val())
 
-viz_cb = VizCallback(test_func, img_gen.next_val())
+    model.fit_generator(generator=img_gen.next_train(), samples_per_epoch=(words_per_epoch - val_words),
+                        nb_epoch=stop_epoch, validation_data=img_gen.next_val(), nb_val_samples=val_words,
+                        callbacks=[viz_cb, img_gen], initial_epoch=start_epoch)
 
-model.fit_generator(generator=img_gen.next_train(), samples_per_epoch=(words_per_epoch - val_words),
-                    nb_epoch=nb_epoch, validation_data=img_gen.next_val(), nb_val_samples=val_words,
-                    callbacks=[viz_cb, img_gen])
+
+if __name__ == '__main__':
+    run_name = datetime.datetime.now().strftime('%Y:%m:%d:%H:%M:%S')
+    train(run_name, 0, 20, 128)
+    # increase to wider images and start at epoch 20. The learned weights are reloaded
+    train(run_name, 20, 25, 512)

examples/imdb_bidirectional_lstm.py

@@ -10,7 +10,7 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
 from keras.models import Sequential
-from keras.layers import Dense, Dropout, Embedding, LSTM, Input, Bidirectional
+from keras.layers import Dense, Dropout, Embedding, LSTM, Bidirectional
 from keras.datasets import imdb
 
 

examples/imdb_cnn.py

@@ -12,11 +12,10 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
 from keras.models import Sequential
-from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Dense, Dropout, Activation
 from keras.layers import Embedding
-from keras.layers import Convolution1D, MaxPooling1D
+from keras.layers import Convolution1D, GlobalMaxPooling1D
 from keras.datasets import imdb
-from keras import backend as K
 
 
 # set parameters:
@@ -58,11 +57,7 @@ model.add(Convolution1D(nb_filter=nb_filter,
                         activation='relu',
                         subsample_length=1))
 # we use max pooling:
-model.add(MaxPooling1D(pool_length=model.output_shape[1]))
-
-# We flatten the output of the conv layer,
-# so that we can add a vanilla dense layer:
-model.add(Flatten())
+model.add(GlobalMaxPooling1D())
 
 # We add a vanilla hidden layer:
 model.add(Dense(hidden_dims))

examples/imdb_cnn_lstm.py

@@ -11,7 +11,7 @@ from keras.preprocessing import sequence
 from keras.models import Sequential
 from keras.layers import Dense, Dropout, Activation
 from keras.layers import Embedding
-from keras.layers import LSTM, GRU, SimpleRNN
+from keras.layers import LSTM
 from keras.layers import Convolution1D, MaxPooling1D
 from keras.datasets import imdb
 

examples/imdb_fasttext.py

@@ -6,8 +6,8 @@ Bags of Tricks for Efficient Text Classification
 https://arxiv.org/abs/1607.01759
 
 Results on IMDB datasets with uni and bi-gram embeddings:
-    Uni-gram: 0.8813 test accuracy after 5 epochs. 15s/epoch on i7 cpu.
-    Bi-gram : 0.9056 test accuracy after 5 epochs. 5s/epoch on GTX 1080 gpu.
+    Uni-gram: 0.8813 test accuracy after 5 epochs. 8s/epoch on i7 cpu.
+    Bi-gram : 0.9056 test accuracy after 5 epochs. 2s/epoch on GTX 980M gpu.
 '''
 
 from __future__ import print_function
@@ -16,9 +16,9 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
 from keras.models import Sequential
-from keras.layers import Dense, Flatten
+from keras.layers import Dense
 from keras.layers import Embedding
-from keras.layers import AveragePooling1D
+from keras.layers import GlobalAveragePooling1D
 from keras.datasets import imdb
 
 
@@ -54,9 +54,9 @@ def add_ngram(sequences, token_indice, ngram_range=2):
     new_sequences = []
     for input_list in sequences:
         new_list = input_list[:]
-        for i in range(len(new_list)-ngram_range+1):
-            for ngram_value in range(2, ngram_range+1):
-                ngram = tuple(new_list[i:i+ngram_value])
+        for i in range(len(new_list) - ngram_range + 1):
+            for ngram_value in range(2, ngram_range + 1):
+                ngram = tuple(new_list[i:i + ngram_value])
                 if ngram in token_indice:
                     new_list.append(token_indice[ngram])
         new_sequences.append(new_list)
@@ -84,7 +84,7 @@ if ngram_range > 1:
     # Create set of unique n-gram from the training set.
     ngram_set = set()
     for input_list in X_train:
-        for i in range(2, ngram_range+1):
+        for i in range(2, ngram_range + 1):
             set_of_ngram = create_ngram_set(input_list, ngram_value=i)
             ngram_set.update(set_of_ngram)
 
@@ -92,7 +92,7 @@ if ngram_range > 1:
     # Integer values are greater than max_features in order
     # to avoid collision with existing features.
     start_index = max_features + 1
-    token_indice = {v: k+start_index for k, v in enumerate(ngram_set)}
+    token_indice = {v: k + start_index for k, v in enumerate(ngram_set)}
     indice_token = {token_indice[k]: k for k in token_indice}
 
     # max_features is the highest integer that could be found in the dataset.
@@ -119,12 +119,9 @@ model.add(Embedding(max_features,
                     embedding_dims,
                     input_length=maxlen))
 
-# we add a AveragePooling1D, which will average the embeddings
+# we add a GlobalAveragePooling1D, which will average the embeddings
 # of all words in the document
-model.add(AveragePooling1D(pool_length=model.output_shape[1]))
-
-# We flatten the output of the AveragePooling1D layer
-model.add(Flatten())
+model.add(GlobalAveragePooling1D())
 
 # We project onto a single unit output layer, and squash it with a sigmoid:
 model.add(Dense(1, activation='sigmoid'))

examples/imdb_lstm.py

@@ -15,10 +15,9 @@ import numpy as np
 np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
-from keras.utils import np_utils
 from keras.models import Sequential
-from keras.layers import Dense, Dropout, Activation, Embedding
-from keras.layers import LSTM, SimpleRNN, GRU
+from keras.layers import Dense, Activation, Embedding
+from keras.layers import LSTM
 from keras.datasets import imdb
 
 max_features = 20000

examples/lstm_text_generation.py

@@ -12,7 +12,7 @@ has at least ~100k characters. ~1M is better.
 
 from __future__ import print_function
 from keras.models import Sequential
-from keras.layers import Dense, Activation, Dropout
+from keras.layers import Dense, Activation
 from keras.layers import LSTM
 from keras.optimizers import RMSprop
 from keras.utils.data_utils import get_file

examples/mnist_acgan.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Train an Auxiliary Classifier Generative Adversarial Network (ACGAN) on the
+MNIST dataset. See https://arxiv.org/abs/1610.09585 for more details.
+
+You should start to see reasonable images after ~5 epochs, and good images
+by ~15 epochs. You should use a GPU, as the convolution-heavy operations are
+very slow on the CPU. Prefer the TensorFlow backend if you plan on iterating, as
+the compilation time can be a blocker using Theano.
+
+Timings:
+
+Hardware           | Backend | Time / Epoch
+-------------------------------------------
+ CPU               | TF      | 3 hrs
+ Titan X (maxwell) | TF      | 4 min
+ Titan X (maxwell) | TH      | 7 min
+
+Consult https://github.com/lukedeo/keras-acgan for more information and
+example output
+"""
+from __future__ import print_function
+
+from collections import defaultdict
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+from PIL import Image
+
+from six.moves import range
+
+import keras.backend as K
+from keras.datasets import mnist
+from keras.layers import Input, Dense, Reshape, Flatten, Embedding, merge, Dropout
+from keras.layers.advanced_activations import LeakyReLU
+from keras.layers.convolutional import UpSampling2D, Convolution2D
+from keras.models import Sequential, Model
+from keras.optimizers import Adam
+from keras.utils.generic_utils import Progbar
+import numpy as np
+
+np.random.seed(1337)
+
+K.set_image_dim_ordering('th')
+
+
+def build_generator(latent_size):
+    # we will map a pair of (z, L), where z is a latent vector and L is a
+    # label drawn from P_c, to image space (..., 1, 28, 28)
+    cnn = Sequential()
+
+    cnn.add(Dense(1024, input_dim=latent_size, activation='relu'))
+    cnn.add(Dense(128 * 7 * 7, activation='relu'))
+    cnn.add(Reshape((128, 7, 7)))
+
+    # upsample to (..., 14, 14)
+    cnn.add(UpSampling2D(size=(2, 2)))
+    cnn.add(Convolution2D(256, 5, 5, border_mode='same',
+                          activation='relu', init='glorot_normal'))
+
+    # upsample to (..., 28, 28)
+    cnn.add(UpSampling2D(size=(2, 2)))
+    cnn.add(Convolution2D(128, 5, 5, border_mode='same',
+                          activation='relu', init='glorot_normal'))
+
+    # take a channel axis reduction
+    cnn.add(Convolution2D(1, 2, 2, border_mode='same',
+                          activation='tanh', init='glorot_normal'))
+
+    # this is the z space commonly refered to in GAN papers
+    latent = Input(shape=(latent_size, ))
+
+    # this will be our label
+    image_class = Input(shape=(1,), dtype='int32')
+
+    # 10 classes in MNIST
+    cls = Flatten()(Embedding(10, latent_size,
+                              init='glorot_normal')(image_class))
+
+    # hadamard product between z-space and a class conditional embedding
+    h = merge([latent, cls], mode='mul')
+
+    fake_image = cnn(h)
+
+    return Model(input=[latent, image_class], output=fake_image)
+
+
+def build_discriminator():
+    # build a relatively standard conv net, with LeakyReLUs as suggested in
+    # the reference paper
+    cnn = Sequential()
+
+    cnn.add(Convolution2D(32, 3, 3, border_mode='same', subsample=(2, 2),
+                          input_shape=(1, 28, 28)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(64, 3, 3, border_mode='same', subsample=(1, 1)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(128, 3, 3, border_mode='same', subsample=(2, 2)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(256, 3, 3, border_mode='same', subsample=(1, 1)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Flatten())
+
+    image = Input(shape=(1, 28, 28))
+
+    features = cnn(image)
+
+    # first output (name=generation) is whether or not the discriminator
+    # thinks the image that is being shown is fake, and the second output
+    # (name=auxiliary) is the class that the discriminator thinks the image
+    # belongs to.
+    fake = Dense(1, activation='sigmoid', name='generation')(features)
+    aux = Dense(10, activation='softmax', name='auxiliary')(features)
+
+    return Model(input=image, output=[fake, aux])
+
+if __name__ == '__main__':
+
+    # batch and latent size taken from the paper
+    nb_epochs = 50
+    batch_size = 100
+    latent_size = 100
+
+    # Adam parameters suggested in https://arxiv.org/abs/1511.06434
+    adam_lr = 0.0002
+    adam_beta_1 = 0.5
+
+    # build the discriminator
+    discriminator = build_discriminator()
+    discriminator.compile(
+        optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+        loss=['binary_crossentropy', 'sparse_categorical_crossentropy']
+    )
+
+    # build the generator
+    generator = build_generator(latent_size)
+    generator.compile(optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+                      loss='binary_crossentropy')
+
+    latent = Input(shape=(latent_size, ))
+    image_class = Input(shape=(1,), dtype='int32')
+
+    # get a fake image
+    fake = generator([latent, image_class])
+
+    # we only want to be able to train generation for the combined model
+    discriminator.trainable = False
+    fake, aux = discriminator(fake)
+    combined = Model(input=[latent, image_class], output=[fake, aux])
+
+    combined.compile(
+        optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+        loss=['binary_crossentropy', 'sparse_categorical_crossentropy']
+    )
+
+    # get our mnist data, and force it to be of shape (..., 1, 28, 28) with
+    # range [-1, 1]
+    (X_train, y_train), (X_test, y_test) = mnist.load_data()
+    X_train = (X_train.astype(np.float32) - 127.5) / 127.5
+    X_train = np.expand_dims(X_train, axis=1)
+
+    X_test = (X_test.astype(np.float32) - 127.5) / 127.5
+    X_test = np.expand_dims(X_test, axis=1)
+
+    nb_train, nb_test = X_train.shape[0], X_test.shape[0]
+
+    train_history = defaultdict(list)
+    test_history = defaultdict(list)
+
+    for epoch in range(nb_epochs):
+        print('Epoch {} of {}'.format(epoch + 1, nb_epochs))
+
+        nb_batches = int(X_train.shape[0] / batch_size)
+        progress_bar = Progbar(target=nb_batches)
+
+        epoch_gen_loss = []
+        epoch_disc_loss = []
+
+        for index in range(nb_batches):
+            progress_bar.update(index)
+            # generate a new batch of noise
+            noise = np.random.uniform(-1, 1, (batch_size, latent_size))
+
+            # get a batch of real images
+            image_batch = X_train[index * batch_size:(index + 1) * batch_size]
+            label_batch = y_train[index * batch_size:(index + 1) * batch_size]
+
+            # sample some labels from p_c
+            sampled_labels = np.random.randint(0, 10, batch_size)
+
+            # generate a batch of fake images, using the generated labels as a
+            # conditioner. We reshape the sampled labels to be
+            # (batch_size, 1) so that we can feed them into the embedding
+            # layer as a length one sequence
+            generated_images = generator.predict(
+                [noise, sampled_labels.reshape((-1, 1))], verbose=0)
+
+            X = np.concatenate((image_batch, generated_images))
+            y = np.array([1] * batch_size + [0] * batch_size)
+            aux_y = np.concatenate((label_batch, sampled_labels), axis=0)
+
+            # see if the discriminator can figure itself out...
+            epoch_disc_loss.append(discriminator.train_on_batch(X, [y, aux_y]))
+
+            # make new noise. we generate 2 * batch size here such that we have
+            # the generator optimize over an identical number of images as the
+            # discriminator
+            noise = np.random.uniform(-1, 1, (2 * batch_size, latent_size))
+            sampled_labels = np.random.randint(0, 10, 2 * batch_size)
+
+            # we want to train the genrator to trick the discriminator
+            # For the generator, we want all the {fake, not-fake} labels to say
+            # not-fake
+            trick = np.ones(2 * batch_size)
+
+            epoch_gen_loss.append(combined.train_on_batch(
+                [noise, sampled_labels.reshape((-1, 1))], [trick, sampled_labels]))
+
+        print('\nTesting for epoch {}:'.format(epoch + 1))
+
+        # evaluate the testing loss here
+
+        # generate a new batch of noise
+        noise = np.random.uniform(-1, 1, (nb_test, latent_size))
+
+        # sample some labels from p_c and generate images from them
+        sampled_labels = np.random.randint(0, 10, nb_test)
+        generated_images = generator.predict(
+            [noise, sampled_labels.reshape((-1, 1))], verbose=False)
+
+        X = np.concatenate((X_test, generated_images))
+        y = np.array([1] * nb_test + [0] * nb_test)
+        aux_y = np.concatenate((y_test, sampled_labels), axis=0)
+
+        # see if the discriminator can figure itself out...
+        discriminator_test_loss = discriminator.evaluate(
+            X, [y, aux_y], verbose=False)
+
+        discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0)
+
+        # make new noise
+        noise = np.random.uniform(-1, 1, (2 * nb_test, latent_size))
+        sampled_labels = np.random.randint(0, 10, 2 * nb_test)
+
+        trick = np.ones(2 * nb_test)
+
+        generator_test_loss = combined.evaluate(
+            [noise, sampled_labels.reshape((-1, 1))],
+            [trick, sampled_labels], verbose=False)
+
+        generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0)
+
+        # generate an epoch report on performance
+        train_history['generator'].append(generator_train_loss)
+        train_history['discriminator'].append(discriminator_train_loss)
+
+        test_history['generator'].append(generator_test_loss)
+        test_history['discriminator'].append(discriminator_test_loss)
+
+        print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}'.format(
+            'component', *discriminator.metrics_names))
+        print('-' * 65)
+
+        ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}'
+        print(ROW_FMT.format('generator (train)',
+                             *train_history['generator'][-1]))
+        print(ROW_FMT.format('generator (test)',
+                             *test_history['generator'][-1]))
+        print(ROW_FMT.format('discriminator (train)',
+                             *train_history['discriminator'][-1]))
+        print(ROW_FMT.format('discriminator (test)',
+                             *test_history['discriminator'][-1]))
+
+        # save weights every epoch
+        generator.save_weights(
+            'params_generator_epoch_{0:03d}.hdf5'.format(epoch), True)
+        discriminator.save_weights(
+            'params_discriminator_epoch_{0:03d}.hdf5'.format(epoch), True)
+
+        # generate some digits to display
+        noise = np.random.uniform(-1, 1, (100, latent_size))
+
+        sampled_labels = np.array([
+            [i] * 10 for i in range(10)
+        ]).reshape(-1, 1)
+
+        # get a batch to display
+        generated_images = generator.predict(
+            [noise, sampled_labels], verbose=0)
+
+        # arrange them into a grid
+        img = (np.concatenate([r.reshape(-1, 28)
+                               for r in np.split(generated_images, 10)
+                               ], axis=-1) * 127.5 + 127.5).astype(np.uint8)
+
+        Image.fromarray(img).save(
+            'plot_epoch_{0:03d}_generated.png'.format(epoch))
+
+    pickle.dump({'train': train_history, 'test': test_history},
+                open('acgan-history.pkl', 'wb'))

examples/mnist_hierarchical_rnn.py

@@ -8,7 +8,7 @@ document vector is considered to preserve both the word-level and
 sentence-level structure of the context.
 
 # References
-    - [A Hierarchical Neural Autoencoder for Paragraphs and Documents](https://web.stanford.edu/~jurafsky/pubs/P15-1107.pdf)
+    - [A Hierarchical Neural Autoencoder for Paragraphs and Documents](https://arxiv.org/abs/1506.01057)
         Encodes paragraphs and documents with HRNN.
         Results have shown that HRNN outperforms standard
         RNNs and may play some role in more sophisticated generation tasks like
@@ -27,7 +27,7 @@ After 5 epochs: train acc: 0.9858, val acc: 0.9864
 from __future__ import print_function
 
 from keras.datasets import mnist
-from keras.models import Sequential, Model
+from keras.models import Model
 from keras.layers import Input, Dense, TimeDistributed
 from keras.layers import LSTM
 from keras.utils import np_utils

examples/mnist_mlp.py

@@ -12,7 +12,7 @@ np.random.seed(1337)  # for reproducibility
 from keras.datasets import mnist
 from keras.models import Sequential
 from keras.layers.core import Dense, Dropout, Activation
-from keras.optimizers import SGD, Adam, RMSprop
+from keras.optimizers import RMSprop
 from keras.utils import np_utils
 
 

examples/mnist_net2net.py

@@ -55,6 +55,7 @@ Results
 '''
 
 from __future__ import print_function
+from six.moves import xrange
 import numpy as np
 np.random.seed(1337)
 

examples/mnist_siamese_graph.py

@@ -19,7 +19,7 @@ import random
 from keras.datasets import mnist
 from keras.models import Sequential, Model
 from keras.layers import Dense, Dropout, Input, Lambda
-from keras.optimizers import SGD, RMSprop
+from keras.optimizers import RMSprop
 from keras import backend as K
 
 
@@ -50,7 +50,7 @@ def create_pairs(x, digit_indices):
     n = min([len(digit_indices[d]) for d in range(10)]) - 1
     for d in range(10):
         for i in range(n):
-            z1, z2 = digit_indices[d][i], digit_indices[d][i+1]
+            z1, z2 = digit_indices[d][i], digit_indices[d][i + 1]
             pairs += [[x[z1], x[z2]]]
             inc = random.randrange(1, 10)
             dn = (d + inc) % 10

examples/mnist_sklearn_wrapper.py

@@ -13,6 +13,7 @@ from keras.layers import Dense, Dropout, Activation, Flatten
 from keras.layers import Convolution2D, MaxPooling2D
 from keras.utils import np_utils
 from keras.wrappers.scikit_learn import KerasClassifier
+from keras import backend as K
 from sklearn.grid_search import GridSearchCV
 
 
@@ -23,8 +24,16 @@ img_rows, img_cols = 28, 28
 
 # load training data and do basic data normalization
 (X_train, y_train), (X_test, y_test) = mnist.load_data()
-X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
-X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
+
+if K.image_dim_ordering() == 'th':
+    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
+    X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
+    input_shape = (1, img_rows, img_cols)
+else:
+    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
+    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
+    input_shape = (img_rows, img_cols, 1)
+
 X_train = X_train.astype('float32')
 X_test = X_test.astype('float32')
 X_train /= 255
@@ -34,6 +43,7 @@ X_test /= 255
 y_train = np_utils.to_categorical(y_train, nb_classes)
 y_test = np_utils.to_categorical(y_test, nb_classes)
 
+
 def make_model(dense_layer_sizes, nb_filters, nb_conv, nb_pool):
     '''Creates model comprised of 2 convolutional layers followed by dense layers
 
@@ -47,7 +57,7 @@ def make_model(dense_layer_sizes, nb_filters, nb_conv, nb_pool):
 
     model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
                             border_mode='valid',
-                            input_shape=(1, img_rows, img_cols)))
+                            input_shape=input_shape))
     model.add(Activation('relu'))
     model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
     model.add(Activation('relu'))

examples/mnist_swwae.py

@@ -44,7 +44,6 @@ Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
 https://arxiv.org/abs/1603.05027v3
 
 '''
-
 from __future__ import print_function
 import numpy as np
 np.random.seed(1337)  # for reproducibility
@@ -76,6 +75,16 @@ def getwhere(x):
     y_prepool, y_postpool = x
     return K.gradients(K.sum(y_postpool), y_prepool)
 
+if K.backend() == 'tensorflow':
+    raise RuntimeError('This example can only run with the '
+                       'Theano backend for the time being, '
+                       'because it requires taking the gradient '
+                       'of a gradient, which isn\'t '
+                       'supported for all TF ops.')
+
+# This example assume 'th' dim ordering.
+K.set_image_dim_ordering('th')
+
 # input image dimensions
 img_rows, img_cols = 28, 28
 

examples/neural_doodle.py

@@ -47,7 +47,7 @@ from scipy.optimize import fmin_l_bfgs_b
 from scipy.misc import imread, imsave
 
 from keras import backend as K
-from keras.layers import Input, Convolution2D, MaxPooling2D, AveragePooling2D
+from keras.layers import Input, AveragePooling2D
 from keras.models import Model
 from keras.preprocessing.image import load_img, img_to_array
 from keras.applications import vgg19
@@ -108,10 +108,12 @@ def deprocess_image(x):
         x = x.transpose((1, 2, 0))
     else:
         x = x.reshape((img_nrows, img_ncols, 3))
-    x = x[:, :, ::-1]
+    # Remove zero-center by mean pixel
     x[:, :, 0] += 103.939
     x[:, :, 1] += 116.779
     x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
@@ -299,7 +301,7 @@ loss_grads = K.gradients(loss, target_image)
 
 # Evaluator class for computing efficiency
 outputs = [loss]
-if type(loss_grads) in {list, tuple}:
+if isinstance(loss_grads, (list, tuple)):
     outputs += loss_grads
 else:
     outputs.append(loss_grads)

examples/neural_style_transfer.py

@@ -8,6 +8,13 @@ e.g.:
 ```
 python neural_style_transfer.py img/tuebingen.jpg img/starry_night.jpg results/my_result
 ```
+Optional parameters:
+```
+--iter, To specify the number of iterations the style transfer takes place (Default is 10)
+--content_weight, The weight given to the content loss (Default is 0.025)
+--style_weight, The weight given to the style loss (Default is 1.0)
+--tv_weight, The weight given to the total variation loss (Default is 1.0)
+```
 
 It is preferable to run this script on GPU, for speed.
 
@@ -60,23 +67,34 @@ parser.add_argument('style_reference_image_path', metavar='ref', type=str,
                     help='Path to the style reference image.')
 parser.add_argument('result_prefix', metavar='res_prefix', type=str,
                     help='Prefix for the saved results.')
+parser.add_argument('--iter', type=int, default=10, required=False,
+                    help='Number of iterations to run.')
+parser.add_argument('--content_weight', type=float, default=0.025, required=False,
+                    help='Content weight.')
+parser.add_argument('--style_weight', type=float, default=1.0, required=False,
+                    help='Style weight.')
+parser.add_argument('--tv_weight', type=float, default=1.0, required=False,
+                    help='Total Variation weight.')
 
 args = parser.parse_args()
 base_image_path = args.base_image_path
 style_reference_image_path = args.style_reference_image_path
 result_prefix = args.result_prefix
+iterations = args.iter
 
 # these are the weights of the different loss components
-total_variation_weight = 1.
-style_weight = 1.
-content_weight = 0.025
+total_variation_weight = args.tv_weight
+style_weight = args.style_weight
+content_weight = args.content_weight
 
 # dimensions of the generated picture.
+width, height = load_img(base_image_path).size
 img_nrows = 400
-img_ncols = 400
-assert img_ncols == img_nrows, 'Due to the use of the Gram matrix, width and height must match.'
+img_ncols = int(width * img_nrows / height)
 
 # util function to open, resize and format pictures into appropriate tensors
+
+
 def preprocess_image(image_path):
     img = load_img(image_path, target_size=(img_nrows, img_ncols))
     img = img_to_array(img)
@@ -85,16 +103,20 @@ def preprocess_image(image_path):
     return img
 
 # util function to convert a tensor into a valid image
+
+
 def deprocess_image(x):
     if K.image_dim_ordering() == 'th':
         x = x.reshape((3, img_nrows, img_ncols))
         x = x.transpose((1, 2, 0))
     else:
         x = x.reshape((img_nrows, img_ncols, 3))
-    x = x[:, :, ::-1]
+    # Remove zero-center by mean pixel
     x[:, :, 0] += 103.939
     x[:, :, 1] += 116.779
     x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
@@ -126,6 +148,8 @@ outputs_dict = dict([(layer.name, layer.output) for layer in model.layers])
 # first we need to define 4 util functions
 
 # the gram matrix of an image tensor (feature-wise outer product)
+
+
 def gram_matrix(x):
     assert K.ndim(x) == 3
     if K.image_dim_ordering() == 'th':
@@ -140,6 +164,8 @@ def gram_matrix(x):
 # It is based on the gram matrices (which capture style) of
 # feature maps from the style reference image
 # and from the generated image
+
+
 def style_loss(style, combination):
     assert K.ndim(style) == 3
     assert K.ndim(combination) == 3
@@ -152,19 +178,23 @@ def style_loss(style, combination):
 # an auxiliary loss function
 # designed to maintain the "content" of the
 # base image in the generated image
+
+
 def content_loss(base, combination):
     return K.sum(K.square(combination - base))
 
 # the 3rd loss function, total variation loss,
 # designed to keep the generated image locally coherent
+
+
 def total_variation_loss(x):
     assert K.ndim(x) == 4
     if K.image_dim_ordering() == 'th':
-        a = K.square(x[:, :, :img_nrows-1, :img_ncols-1] - x[:, :, 1:, :img_ncols-1])
-        b = K.square(x[:, :, :img_nrows-1, :img_ncols-1] - x[:, :, :img_nrows-1, 1:])
+        a = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, 1:, :img_ncols - 1])
+        b = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, :img_nrows - 1, 1:])
     else:
-        a = K.square(x[:, :img_nrows-1, :img_ncols-1, :] - x[:, 1:, :img_ncols-1, :])
-        b = K.square(x[:, :img_nrows-1, :img_ncols-1, :] - x[:, :img_nrows-1, 1:, :])
+        a = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, 1:, :img_ncols - 1, :])
+        b = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, :img_nrows - 1, 1:, :])
     return K.sum(K.pow(a + b, 1.25))
 
 # combine these loss functions into a single scalar
@@ -190,13 +220,14 @@ loss += total_variation_weight * total_variation_loss(combination_image)
 grads = K.gradients(loss, combination_image)
 
 outputs = [loss]
-if type(grads) in {list, tuple}:
+if isinstance(grads, (list, tuple)):
     outputs += grads
 else:
     outputs.append(grads)
 
 f_outputs = K.function([combination_image], outputs)
 
+
 def eval_loss_and_grads(x):
     if K.image_dim_ordering() == 'th':
         x = x.reshape((1, 3, img_nrows, img_ncols))
@@ -216,7 +247,10 @@ def eval_loss_and_grads(x):
 # "loss" and "grads". This is done because scipy.optimize
 # requires separate functions for loss and gradients,
 # but computing them separately would be inefficient.
+
+
 class Evaluator(object):
+
     def __init__(self):
         self.loss_value = None
         self.grads_values = None
@@ -244,7 +278,7 @@ if K.image_dim_ordering() == 'th':
 else:
     x = np.random.uniform(0, 255, (1, img_nrows, img_ncols, 3)) - 128.
 
-for i in range(10):
+for i in range(iterations):
     print('Start of iteration', i)
     start_time = time.time()
     x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(),

examples/pretrained_word_embeddings.py

@@ -102,9 +102,9 @@ print('Preparing embedding matrix.')
 
 # prepare embedding matrix
 nb_words = min(MAX_NB_WORDS, len(word_index))
-embedding_matrix = np.zeros((nb_words + 1, EMBEDDING_DIM))
+embedding_matrix = np.zeros((nb_words, EMBEDDING_DIM))
 for word, i in word_index.items():
-    if i > MAX_NB_WORDS:
+    if i >= MAX_NB_WORDS:
         continue
     embedding_vector = embeddings_index.get(word)
     if embedding_vector is not None:
@@ -113,7 +113,7 @@ for word, i in word_index.items():
 
 # load pre-trained word embeddings into an Embedding layer
 # note that we set trainable = False so as to keep the embeddings fixed
-embedding_layer = Embedding(nb_words + 1,
+embedding_layer = Embedding(nb_words,
                             EMBEDDING_DIM,
                             weights=[embedding_matrix],
                             input_length=MAX_SEQUENCE_LENGTH,

examples/reuters_mlp.py

@@ -21,7 +21,7 @@ print('Loading data...')
 print(len(X_train), 'train sequences')
 print(len(X_test), 'test sequences')
 
-nb_classes = np.max(y_train)+1
+nb_classes = np.max(y_train) + 1
 print(nb_classes, 'classes')
 
 print('Vectorizing sequence data...')

examples/stateful_lstm.py

@@ -54,7 +54,6 @@ model.add(LSTM(50,
                return_sequences=True,
                stateful=True))
 model.add(LSTM(50,
-               batch_input_shape=(batch_size, tsteps, 1),
                return_sequences=False,
                stateful=True))
 model.add(Dense(1))

examples/variational_autoencoder.py

@@ -4,6 +4,7 @@ Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
 '''
 import numpy as np
 import matplotlib.pyplot as plt
+from scipy.stats import norm
 
 from keras.layers import Input, Dense, Lambda
 from keras.models import Model
@@ -16,6 +17,7 @@ original_dim = 784
 latent_dim = 2
 intermediate_dim = 256
 nb_epoch = 50
+epsilon_std = 1.0
 
 x = Input(batch_shape=(batch_size, original_dim))
 h = Dense(intermediate_dim, activation='relu')(x)
@@ -25,7 +27,8 @@ z_log_var = Dense(latent_dim)(h)
 
 def sampling(args):
     z_mean, z_log_var = args
-    epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0.)
+    epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0.,
+                              std=epsilon_std)
     return z_mean + K.exp(z_log_var / 2) * epsilon
 
 # note that "output_shape" isn't necessary with the TensorFlow backend
@@ -80,9 +83,10 @@ generator = Model(decoder_input, _x_decoded_mean)
 n = 15  # figure with 15x15 digits
 digit_size = 28
 figure = np.zeros((digit_size * n, digit_size * n))
-# we will sample n points within [-15, 15] standard deviations
-grid_x = np.linspace(-15, 15, n)
-grid_y = np.linspace(-15, 15, n)
+# linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
+# to produce values of the latent variables z, since the prior of the latent space is Gaussian
+grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
+grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
 
 for i, yi in enumerate(grid_x):
     for j, xi in enumerate(grid_y):
@@ -93,5 +97,5 @@ for i, yi in enumerate(grid_x):
                j * digit_size: (j + 1) * digit_size] = digit
 
 plt.figure(figsize=(10, 10))
-plt.imshow(figure)
+plt.imshow(figure, cmap='Greys_r')
 plt.show()

examples/variational_autoencoder_deconv.py

@@ -1,12 +1,14 @@
-'''This script demonstrates how to build a variational autoencoder with Keras and deconvolution layers.
+'''This script demonstrates how to build a variational autoencoder
+with Keras and deconvolution layers.
 
 Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
 '''
 import numpy as np
 import matplotlib.pyplot as plt
+from scipy.stats import norm
 
 from keras.layers import Input, Dense, Lambda, Flatten, Reshape
-from keras.layers import Convolution2D, Deconvolution2D, MaxPooling2D
+from keras.layers import Convolution2D, Deconvolution2D
 from keras.models import Model
 from keras import backend as K
 from keras import objectives
@@ -15,25 +17,36 @@ from keras.datasets import mnist
 # input image dimensions
 img_rows, img_cols, img_chns = 28, 28, 1
 # number of convolutional filters to use
-nb_filters = 32
+nb_filters = 64
 # convolution kernel size
 nb_conv = 3
 
-batch_size = 16
-original_dim = (img_chns, img_rows, img_cols)
+batch_size = 100
+if K.image_dim_ordering() == 'th':
+    original_img_size = (img_chns, img_rows, img_cols)
+else:
+    original_img_size = (img_rows, img_cols, img_chns)
 latent_dim = 2
 intermediate_dim = 128
-epsilon_std = 0.01
+epsilon_std = 1.0
 nb_epoch = 5
 
+x = Input(batch_shape=(batch_size,) + original_img_size)
+conv_1 = Convolution2D(img_chns, 2, 2, border_mode='same', activation='relu')(x)
+conv_2 = Convolution2D(nb_filters, 2, 2,
+                       border_mode='same', activation='relu',
+                       subsample=(2, 2))(conv_1)
+conv_3 = Convolution2D(nb_filters, nb_conv, nb_conv,
+                       border_mode='same', activation='relu',
+                       subsample=(1, 1))(conv_2)
+conv_4 = Convolution2D(nb_filters, nb_conv, nb_conv,
+                       border_mode='same', activation='relu',
+                       subsample=(1, 1))(conv_3)
+flat = Flatten()(conv_4)
+hidden = Dense(intermediate_dim, activation='relu')(flat)
 
-x = Input(batch_shape=(batch_size,) + original_dim)
-c = Convolution2D(nb_filters, nb_conv, nb_conv, border_mode='same', activation='relu')(x)
-f = Flatten()(c)
-h = Dense(intermediate_dim, activation='relu')(f)
-
-z_mean = Dense(latent_dim)(h)
-z_log_var = Dense(latent_dim)(h)
+z_mean = Dense(latent_dim)(hidden)
+z_log_var = Dense(latent_dim)(hidden)
 
 
 def sampling(args):
@@ -47,36 +60,69 @@ def sampling(args):
 z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
 
 # we instantiate these layers separately so as to reuse them later
-decoder_h = Dense(intermediate_dim, activation='relu')
-decoder_f = Dense(nb_filters*img_rows*img_cols, activation='relu')
-decoder_c = Reshape((nb_filters, img_rows, img_cols))
-decoder_mean = Deconvolution2D(img_chns, nb_conv, nb_conv,
-                               (batch_size, img_chns, img_rows, img_cols),
-                               border_mode='same')
+decoder_hid = Dense(intermediate_dim, activation='relu')
+decoder_upsample = Dense(nb_filters * 14 * 14, activation='relu')
 
-h_decoded = decoder_h(z)
-f_decoded = decoder_f(h_decoded)
-c_decoded = decoder_c(f_decoded)
-x_decoded_mean = decoder_mean(c_decoded)
+if K.image_dim_ordering() == 'th':
+    output_shape = (batch_size, nb_filters, 14, 14)
+else:
+    output_shape = (batch_size, 14, 14, nb_filters)
+
+decoder_reshape = Reshape(output_shape[1:])
+decoder_deconv_1 = Deconvolution2D(nb_filters, nb_conv, nb_conv,
+                                   output_shape,
+                                   border_mode='same',
+                                   subsample=(1, 1),
+                                   activation='relu')
+decoder_deconv_2 = Deconvolution2D(nb_filters, nb_conv, nb_conv,
+                                   output_shape,
+                                   border_mode='same',
+                                   subsample=(1, 1),
+                                   activation='relu')
+if K.image_dim_ordering() == 'th':
+    output_shape = (batch_size, nb_filters, 29, 29)
+else:
+    output_shape = (batch_size, 29, 29, nb_filters)
+decoder_deconv_3_upsamp = Deconvolution2D(nb_filters, 2, 2,
+                                          output_shape,
+                                          border_mode='valid',
+                                          subsample=(2, 2),
+                                          activation='relu')
+decoder_mean_squash = Convolution2D(img_chns, 2, 2,
+                                    border_mode='valid',
+                                    activation='sigmoid')
+
+hid_decoded = decoder_hid(z)
+up_decoded = decoder_upsample(hid_decoded)
+reshape_decoded = decoder_reshape(up_decoded)
+deconv_1_decoded = decoder_deconv_1(reshape_decoded)
+deconv_2_decoded = decoder_deconv_2(deconv_1_decoded)
+x_decoded_relu = decoder_deconv_3_upsamp(deconv_2_decoded)
+x_decoded_mean_squash = decoder_mean_squash(x_decoded_relu)
 
 
 def vae_loss(x, x_decoded_mean):
-    # NOTE: binary_crossentropy expects a batch_size by dim for x and x_decoded_mean, so we MUST flatten these!
+    # NOTE: binary_crossentropy expects a batch_size by dim
+    # for x and x_decoded_mean, so we MUST flatten these!
     x = K.flatten(x)
     x_decoded_mean = K.flatten(x_decoded_mean)
-    xent_loss = objectives.binary_crossentropy(x, x_decoded_mean)
+    xent_loss = img_rows * img_cols * objectives.binary_crossentropy(x, x_decoded_mean)
     kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
     return xent_loss + kl_loss
 
-vae = Model(x, x_decoded_mean)
+vae = Model(x, x_decoded_mean_squash)
 vae.compile(optimizer='rmsprop', loss=vae_loss)
 vae.summary()
 
 # train the VAE on MNIST digits
-(x_train, y_train), (x_test, y_test) = mnist.load_data()
+(x_train, _), (x_test, y_test) = mnist.load_data()
 
-x_train = x_train.astype('float32')[:, None, :, :] / 255.
-x_test = x_test.astype('float32')[:, None, :, :] / 255.
+x_train = x_train.astype('float32') / 255.
+x_train = x_train.reshape((x_train.shape[0],) + original_img_size)
+x_test = x_test.astype('float32') / 255.
+x_test = x_test.reshape((x_test.shape[0],) + original_img_size)
+
+print('x_train.shape:', x_train.shape)
 
 vae.fit(x_train, x_train,
         shuffle=True,
@@ -84,7 +130,6 @@ vae.fit(x_train, x_train,
         batch_size=batch_size,
         validation_data=(x_test, x_test))
 
-
 # build a model to project inputs on the latent space
 encoder = Model(x, z_mean)
 
@@ -97,28 +142,33 @@ plt.show()
 
 # build a digit generator that can sample from the learned distribution
 decoder_input = Input(shape=(latent_dim,))
-_h_decoded = decoder_h(decoder_input)
-_f_decoded = decoder_f(_h_decoded)
-_c_decoded = decoder_c(_f_decoded)
-_x_decoded_mean = decoder_mean(_c_decoded)
-generator = Model(decoder_input, _x_decoded_mean)
+_hid_decoded = decoder_hid(decoder_input)
+_up_decoded = decoder_upsample(_hid_decoded)
+_reshape_decoded = decoder_reshape(_up_decoded)
+_deconv_1_decoded = decoder_deconv_1(_reshape_decoded)
+_deconv_2_decoded = decoder_deconv_2(_deconv_1_decoded)
+_x_decoded_relu = decoder_deconv_3_upsamp(_deconv_2_decoded)
+_x_decoded_mean_squash = decoder_mean_squash(_x_decoded_relu)
+generator = Model(decoder_input, _x_decoded_mean_squash)
 
 # display a 2D manifold of the digits
 n = 15  # figure with 15x15 digits
 digit_size = 28
 figure = np.zeros((digit_size * n, digit_size * n))
-# we will sample n points within [-15, 15] standard deviations
-grid_x = np.linspace(-15, 15, n)
-grid_y = np.linspace(-15, 15, n)
+# linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
+# to produce values of the latent variables z, since the prior of the latent space is Gaussian
+grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
+grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
 
 for i, yi in enumerate(grid_x):
     for j, xi in enumerate(grid_y):
         z_sample = np.array([[xi, yi]])
-        x_decoded = generator.predict(z_sample)
+        z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2)
+        x_decoded = generator.predict(z_sample, batch_size=batch_size)
         digit = x_decoded[0].reshape(digit_size, digit_size)
         figure[i * digit_size: (i + 1) * digit_size,
                j * digit_size: (j + 1) * digit_size] = digit
 
 plt.figure(figsize=(10, 10))
-plt.imshow(figure)
+plt.imshow(figure, cmap='Greys_r')
 plt.show()

keras/__init__.py

@@ -15,4 +15,4 @@ from . import objectives
 from . import optimizers
 from . import regularizers
 
-__version__ = '1.1.0'
+__version__ = '1.2.2'

keras/activations.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def softmax(x):
@@ -11,8 +12,13 @@ def softmax(x):
         s = K.sum(e, axis=-1, keepdims=True)
         return e / s
     else:
-        raise Exception('Cannot apply softmax to a tensor that is not 2D or 3D. ' +
-                        'Here, ndim=' + str(ndim))
+        raise ValueError('Cannot apply softmax to a tensor '
+                         'that is not 2D or 3D. '
+                         'Here, ndim=' + str(ndim))
+
+
+def elu(x, alpha=1.0):
+    return K.elu(x, alpha)
 
 
 def softplus(x):
@@ -40,13 +46,9 @@ def hard_sigmoid(x):
 
 
 def linear(x):
-    '''
-    The function returns the variable that is passed in, so all types work.
-    '''
     return x
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier):
     if identifier is None:
         return linear

keras/applications/__init__.py

@@ -2,3 +2,4 @@ from .vgg16 import VGG16
 from .vgg19 import VGG19
 from .resnet50 import ResNet50
 from .inception_v3 import InceptionV3
+from .xception import Xception

keras/applications/audio_conv_utils.py

@@ -0,0 +1,92 @@
+import numpy as np
+from .. import backend as K
+
+try:
+    import librosa
+except ImportError:
+    librosa = None
+
+
+TAGS = ['rock', 'pop', 'alternative', 'indie', 'electronic',
+        'female vocalists', 'dance', '00s', 'alternative rock', 'jazz',
+        'beautiful', 'metal', 'chillout', 'male vocalists',
+        'classic rock', 'soul', 'indie rock', 'Mellow', 'electronica',
+        '80s', 'folk', '90s', 'chill', 'instrumental', 'punk',
+        'oldies', 'blues', 'hard rock', 'ambient', 'acoustic',
+        'experimental', 'female vocalist', 'guitar', 'Hip-Hop',
+        '70s', 'party', 'country', 'easy listening',
+        'sexy', 'catchy', 'funk', 'electro', 'heavy metal',
+        'Progressive rock', '60s', 'rnb', 'indie pop',
+        'sad', 'House', 'happy']
+
+
+def preprocess_input(audio_path, dim_ordering='default'):
+    """Reads an audio file and outputs a Mel-spectrogram.
+
+    # Arguments
+        audio_path: path to the target audio file.
+        dim_ordering: data format for the output spectrogram image.
+
+    # Returns
+        3D Numpy tensor encoding the Mel-spectrogram.
+
+    # Raises
+        ImportError: if librosa is not available.
+    """
+    if dim_ordering == 'default':
+        dim_ordering = K.image_dim_ordering()
+    assert dim_ordering in {'tf', 'th'}
+
+    if librosa is None:
+        raise ImportError('Librosa is required to process audio files. '
+                          'Install it via `pip install librosa` or visit '
+                          'http://librosa.github.io/librosa/ for details.')
+
+    # mel-spectrogram parameters
+    sr = 12000
+    n_fft = 512
+    n_mels = 96
+    hop_length = 256
+    duration = 29.12
+
+    src, sr = librosa.load(audio_path, sr=sr)
+    n_sample = src.shape[0]
+    n_sample_wanted = int(duration * sr)
+
+    # trim the signal at the center
+    if n_sample < n_sample_wanted:  # if too short
+        src = np.hstack((src, np.zeros((int(duration * sr) - n_sample,))))
+    elif n_sample > n_sample_wanted:  # if too long
+        src = src[(n_sample - n_sample_wanted) // 2:
+                  (n_sample + n_sample_wanted) // 2]
+
+    logam = librosa.logamplitude
+    melgram = librosa.feature.melspectrogram
+    x = logam(melgram(y=src, sr=sr, hop_length=hop_length,
+                      n_fft=n_fft, n_mels=n_mels) ** 2,
+              ref_power=1.0)
+
+    if dim_ordering == 'th':
+        x = np.expand_dims(x, axis=0)
+    elif dim_ordering == 'tf':
+        x = np.expand_dims(x, axis=3)
+    return x
+
+
+def decode_predictions(preds, top_n=5):
+    """Decode the output of a music tagger model.
+
+    # Arguments
+        preds: 2-dimensional numpy array
+        top_n: integer in [0, 50], number of items to show.
+
+    # Returns
+        Decoded output.
+    """
+    assert len(preds.shape) == 2 and preds.shape[1] == 50
+    results = []
+    for pred in preds:
+        result = zip(TAGS, pred)
+        result = sorted(result, key=lambda x: x[1], reverse=True)
+        results.append(result[:top_n])
+    return results

keras/applications/imagenet_utils.py

@@ -9,35 +9,131 @@ CLASS_INDEX_PATH = 'https://s3.amazonaws.com/deep-learning-models/image-models/i
 
 
 def preprocess_input(x, dim_ordering='default'):
+    """Preprocesses a tensor encoding a batch of images.
+
+    # Arguments
+        x: input Numpy tensor, 4D.
+        dim_ordering: data format of the image tensor.
+
+    # Returns
+        Preprocessed tensor.
+    """
     if dim_ordering == 'default':
         dim_ordering = K.image_dim_ordering()
     assert dim_ordering in {'tf', 'th'}
 
     if dim_ordering == 'th':
+        # 'RGB'->'BGR'
+        x = x[:, ::-1, :, :]
+        # Zero-center by mean pixel
         x[:, 0, :, :] -= 103.939
         x[:, 1, :, :] -= 116.779
         x[:, 2, :, :] -= 123.68
-        # 'RGB'->'BGR'
-        x = x[:, ::-1, :, :]
     else:
+        # 'RGB'->'BGR'
+        x = x[:, :, :, ::-1]
+        # Zero-center by mean pixel
         x[:, :, :, 0] -= 103.939
         x[:, :, :, 1] -= 116.779
         x[:, :, :, 2] -= 123.68
-        # 'RGB'->'BGR'
-        x = x[:, :, :, ::-1]
     return x
 
 
-def decode_predictions(preds):
+def decode_predictions(preds, top=5):
+    """Decodes the prediction of an ImageNet model.
+
+    # Arguments
+        preds: Numpy tensor encoding a batch of predictions.
+        top: integer, how many top-guesses to return.
+
+    # Returns
+        A list of lists of top class prediction tuples
+        `(class_name, class_description, score)`.
+        One list of tuples per sample in batch input.
+
+    # Raises
+        ValueError: in case of invalid shape of the `pred` array
+            (must be 2D).
+    """
     global CLASS_INDEX
-    assert len(preds.shape) == 2 and preds.shape[1] == 1000
+    if len(preds.shape) != 2 or preds.shape[1] != 1000:
+        raise ValueError('`decode_predictions` expects '
+                         'a batch of predictions '
+                         '(i.e. a 2D array of shape (samples, 1000)). '
+                         'Found array with shape: ' + str(preds.shape))
     if CLASS_INDEX is None:
         fpath = get_file('imagenet_class_index.json',
                          CLASS_INDEX_PATH,
                          cache_subdir='models')
         CLASS_INDEX = json.load(open(fpath))
-    indices = np.argmax(preds, axis=-1)
     results = []
-    for i in indices:
-        results.append(CLASS_INDEX[str(i)])
+    for pred in preds:
+        top_indices = pred.argsort()[-top:][::-1]
+        result = [tuple(CLASS_INDEX[str(i)]) + (pred[i],) for i in top_indices]
+        result.sort(key=lambda x: x[2], reverse=True)
+        results.append(result)
     return results
+
+
+def _obtain_input_shape(input_shape,
+                        default_size,
+                        min_size,
+                        dim_ordering,
+                        include_top):
+    """Internal utility to compute/validate an ImageNet model's input shape.
+
+    # Arguments
+        input_shape: either None (will return the default network input shape),
+            or a user-provided shape to be validated.
+        default_size: default input width/height for the model.
+        min_size: minimum input width/height accepted by the model.
+        dim_ordering: image data format to use.
+        include_top: whether the model is expected to
+            be linked to a classifier via a Flatten layer.
+
+    # Returns
+        An integer shape tuple (may include None entries).
+
+    # Raises
+        ValueError: in case of invalid argument values.
+    """
+    if dim_ordering == 'th':
+        default_shape = (3, default_size, default_size)
+    else:
+        default_shape = (default_size, default_size, 3)
+    if include_top:
+        if input_shape is not None:
+            if input_shape != default_shape:
+                raise ValueError('When setting`include_top=True`, '
+                                 '`input_shape` should be ' + str(default_shape) + '.')
+        input_shape = default_shape
+    else:
+        if dim_ordering == 'th':
+            if input_shape is not None:
+                if len(input_shape) != 3:
+                    raise ValueError('`input_shape` must be a tuple of three integers.')
+                if input_shape[0] != 3:
+                    raise ValueError('The input must have 3 channels; got '
+                                     '`input_shape=' + str(input_shape) + '`')
+                if ((input_shape[1] is not None and input_shape[1] < min_size) or
+                   (input_shape[2] is not None and input_shape[2] < min_size)):
+                    raise ValueError('Input size must be at least ' +
+                                     str(min_size) + 'x' + str(min_size) + ', got '
+                                     '`input_shape=' + str(input_shape) + '`')
+            else:
+                input_shape = (3, None, None)
+        else:
+            if input_shape is not None:
+                if len(input_shape) != 3:
+                    raise ValueError('`input_shape` must be a tuple of three integers.')
+                if input_shape[-1] != 3:
+                    raise ValueError('The input must have 3 channels; got '
+                                     '`input_shape=' + str(input_shape) + '`')
+                if ((input_shape[0] is not None and input_shape[0] < min_size) or
+                   (input_shape[1] is not None and input_shape[1] < min_size)):
+                    raise ValueError('Input size must be at least ' +
+                                     str(min_size) + 'x' + str(min_size) + ', got '
+                                     '`input_shape=' + str(input_shape) + '`')
+            else:
+                input_shape = (None, None, 3)
+    return input_shape

keras/applications/inception_v3.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-'''Inception V3 model for Keras.
+"""Inception V3 model for Keras.
 
 Note that the ImageNet weights provided are from a model that had not fully converged.
 Inception v3 should be able to reach 6.9% top-5 error, but our model
@@ -7,14 +7,14 @@ only gets to 7.8% (same as a fully-converged ResNet 50).
 For comparison, VGG16 only gets to 9.9%, quite a bit worse.
 
 Also, do note that the input image format for this model is different than for
-other models (299x299 instead of 224x224), and that the input preprocessing function
-is also different.
+the VGG16 and ResNet models (299x299 instead of 224x224), and that the input preprocessing function
+is also different (same as Xception).
 
-# Reference:
+# Reference
 
 - [Rethinking the Inception Architecture for Computer Vision](http://arxiv.org/abs/1512.00567)
 
-'''
+"""
 from __future__ import print_function
 from __future__ import absolute_import
 
@@ -23,10 +23,11 @@ import warnings
 from ..models import Model
 from ..layers import Flatten, Dense, Input, BatchNormalization, merge
 from ..layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from ..engine.topology import get_source_inputs
 from ..utils.layer_utils import convert_all_kernels_in_model
 from ..utils.data_utils import get_file
 from .. import backend as K
-from .imagenet_utils import decode_predictions
+from .imagenet_utils import decode_predictions, _obtain_input_shape
 
 
 TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/inception_v3_weights_th_dim_ordering_th_kernels.h5'
@@ -38,8 +39,8 @@ TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/relea
 def conv2d_bn(x, nb_filter, nb_row, nb_col,
               border_mode='same', subsample=(1, 1),
               name=None):
-    '''Utility function to apply conv + BN.
-    '''
+    """Utility function to apply conv + BN.
+    """
     if name is not None:
         bn_name = name + '_bn'
         conv_name = name + '_conv'
@@ -60,8 +61,9 @@ def conv2d_bn(x, nb_filter, nb_row, nb_col,
 
 
 def InceptionV3(include_top=True, weights='imagenet',
-                input_tensor=None):
-    '''Instantiate the Inception v3 architecture,
+                input_tensor=None, input_shape=None,
+                classes=1000):
+    """Instantiate the Inception v3 architecture,
     optionally loading weights pre-trained
     on ImageNet. Note that when using TensorFlow,
     for best performance you should set
@@ -76,31 +78,41 @@ def InceptionV3(include_top=True, weights='imagenet',
     Note that the default input image size for this model is 299x299.
 
     # Arguments
-        include_top: whether to include the 3 fully-connected
-            layers at the top of the network.
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
         weights: one of `None` (random initialization)
             or "imagenet" (pre-training on ImageNet).
         input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
             to use as image input for the model.
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(299, 299, 3)` (with `tf` dim ordering)
+            or `(3, 299, 299)` (with `th` dim ordering).
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 139.
+            E.g. `(150, 150, 3)` would be one valid value.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
 
     # Returns
         A Keras model instance.
-    '''
+    """
     if weights not in {'imagenet', None}:
         raise ValueError('The `weights` argument should be either '
                          '`None` (random initialization) or `imagenet` '
                          '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as imagenet with `include_top`'
+                         ' as true, `classes` should be 1000')
+
     # Determine proper input shape
-    if K.image_dim_ordering() == 'th':
-        if include_top:
-            input_shape = (3, 299, 299)
-        else:
-            input_shape = (3, None, None)
-    else:
-        if include_top:
-            input_shape = (299, 299, 3)
-        else:
-            input_shape = (None, None, 3)
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=299,
+                                      min_size=139,
+                                      dim_ordering=K.image_dim_ordering(),
+                                      include_top=include_top)
 
     if input_tensor is None:
         img_input = Input(shape=input_shape)
@@ -259,10 +271,16 @@ def InceptionV3(include_top=True, weights='imagenet',
         # Classification block
         x = AveragePooling2D((8, 8), strides=(8, 8), name='avg_pool')(x)
         x = Flatten(name='flatten')(x)
-        x = Dense(1000, activation='softmax', name='predictions')(x)
+        x = Dense(classes, activation='softmax', name='predictions')(x)
 
-    # Create model
-    model = Model(img_input, x)
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+    # Create model.
+    model = Model(inputs, x, name='inception_v3')
 
     # load weights
     if weights == 'imagenet':

keras/applications/music_tagger_crnn.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+"""MusicTaggerCRNN model for Keras.
+
+# Reference:
+
+- [Music-auto_tagging-keras](https://github.com/keunwoochoi/music-auto_tagging-keras)
+
+"""
+from __future__ import print_function
+from __future__ import absolute_import
+
+from .. import backend as K
+from ..layers import Input, Dense
+from ..models import Model
+from ..layers import Dense, Dropout, Reshape, Permute
+from ..layers.convolutional import Convolution2D
+from ..layers.convolutional import MaxPooling2D, ZeroPadding2D
+from ..layers.normalization import BatchNormalization
+from ..layers.advanced_activations import ELU
+from ..layers.recurrent import GRU
+from ..engine.topology import get_source_inputs
+from ..utils.data_utils import get_file
+from ..utils.layer_utils import convert_all_kernels_in_model
+from .audio_conv_utils import decode_predictions, preprocess_input
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.3/music_tagger_crnn_weights_tf_kernels_th_dim_ordering.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.3/music_tagger_crnn_weights_tf_kernels_tf_dim_ordering.h5'
+
+
+def MusicTaggerCRNN(weights='msd', input_tensor=None,
+                    include_top=True, classes=50):
+    """Instantiate the MusicTaggerCRNN architecture,
+    optionally loading weights pre-trained
+    on Million Song Dataset. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    For preparing mel-spectrogram input, see
+    `audio_conv_utils.py` in [applications](https://github.com/fchollet/keras/tree/master/keras/applications).
+    You will need to install [Librosa](http://librosa.github.io/librosa/)
+    to use it.
+
+    # Arguments
+        weights: one of `None` (random initialization)
+            or "msd" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+        include_top: whether to include the 1 fully-connected
+            layer (output layer) at the top of the network.
+            If False, the network outputs 32-dim features.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+
+    # Returns
+        A Keras model instance.
+    """
+    if weights not in {'msd', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `msd` '
+                         '(pre-training on Million Song Dataset).')
+
+    if weights == 'msd' and include_top and classes != 50:
+        raise ValueError('If using `weights` as msd with `include_top`'
+                         ' as true, `classes` should be 50')
+    # Determine proper input shape
+    if K.image_dim_ordering() == 'th':
+        input_shape = (1, 96, 1366)
+    else:
+        input_shape = (96, 1366, 1)
+
+    if input_tensor is None:
+        melgram_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            melgram_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            melgram_input = input_tensor
+
+    # Determine input axis
+    if K.image_dim_ordering() == 'th':
+        channel_axis = 1
+        freq_axis = 2
+        time_axis = 3
+    else:
+        channel_axis = 3
+        freq_axis = 1
+        time_axis = 2
+
+    # Input block
+    x = ZeroPadding2D(padding=(0, 37))(melgram_input)
+    x = BatchNormalization(axis=time_axis, name='bn_0_freq')(x)
+
+    # Conv block 1
+    x = Convolution2D(64, 3, 3, border_mode='same', name='conv1')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn1')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x)
+
+    # Conv block 2
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv2')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn2')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(x)
+
+    # Conv block 3
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv3')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn3')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(x)
+
+    # Conv block 4
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv4')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn4')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(x)
+
+    # reshaping
+    if K.image_dim_ordering() == 'th':
+        x = Permute((3, 1, 2))(x)
+    x = Reshape((15, 128))(x)
+
+    # GRU block 1, 2, output
+    x = GRU(32, return_sequences=True, name='gru1')(x)
+    x = GRU(32, return_sequences=False, name='gru2')(x)
+
+    if include_top:
+        x = Dense(classes, activation='sigmoid', name='output')(x)
+
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = melgram_input
+    # Create model.
+    model = Model(inputs, x, name='music_tagger_crnn')
+
+    if weights is None:
+        return model
+    else:
+        # Load weights
+        if K.image_dim_ordering() == 'tf':
+            weights_path = get_file('music_tagger_crnn_weights_tf_kernels_tf_dim_ordering.h5',
+                                    TF_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        else:
+            weights_path = get_file('music_tagger_crnn_weights_tf_kernels_th_dim_ordering.h5',
+                                    TH_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        model.load_weights(weights_path, by_name=True)
+        if K.backend() == 'theano':
+            convert_all_kernels_in_model(model)
+        return model

keras/applications/resnet50.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
-'''ResNet50 model for Keras.
+"""ResNet50 model for Keras.
 
 # Reference:
 
 - [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385)
 
 Adapted from code contributed by BigMoyan.
-'''
+"""
 from __future__ import print_function
 from __future__ import absolute_import
 
@@ -18,9 +18,10 @@ from ..layers import Convolution2D, MaxPooling2D, ZeroPadding2D, AveragePooling2
 from ..layers import BatchNormalization
 from ..models import Model
 from .. import backend as K
+from ..engine.topology import get_source_inputs
 from ..utils.layer_utils import convert_all_kernels_in_model
 from ..utils.data_utils import get_file
-from .imagenet_utils import decode_predictions, preprocess_input
+from .imagenet_utils import decode_predictions, preprocess_input, _obtain_input_shape
 
 
 TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_th_dim_ordering_th_kernels.h5'
@@ -30,7 +31,7 @@ TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/relea
 
 
 def identity_block(input_tensor, kernel_size, filters, stage, block):
-    '''The identity_block is the block that has no conv layer at shortcut
+    """The identity_block is the block that has no conv layer at shortcut
 
     # Arguments
         input_tensor: input tensor
@@ -38,7 +39,7 @@ def identity_block(input_tensor, kernel_size, filters, stage, block):
         filters: list of integers, the nb_filters of 3 conv layer at main path
         stage: integer, current stage label, used for generating layer names
         block: 'a','b'..., current block label, used for generating layer names
-    '''
+    """
     nb_filter1, nb_filter2, nb_filter3 = filters
     if K.image_dim_ordering() == 'tf':
         bn_axis = 3
@@ -65,7 +66,7 @@ def identity_block(input_tensor, kernel_size, filters, stage, block):
 
 
 def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)):
-    '''conv_block is the block that has a conv layer at shortcut
+    """conv_block is the block that has a conv layer at shortcut
 
     # Arguments
         input_tensor: input tensor
@@ -76,7 +77,7 @@ def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2))
 
     Note that from stage 3, the first conv layer at main path is with subsample=(2,2)
     And the shortcut should have subsample=(2,2) as well
-    '''
+    """
     nb_filter1, nb_filter2, nb_filter3 = filters
     if K.image_dim_ordering() == 'tf':
         bn_axis = 3
@@ -108,8 +109,9 @@ def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2))
 
 
 def ResNet50(include_top=True, weights='imagenet',
-             input_tensor=None):
-    '''Instantiate the ResNet50 architecture,
+             input_tensor=None, input_shape=None,
+             classes=1000):
+    """Instantiate the ResNet50 architecture,
     optionally loading weights pre-trained
     on ImageNet. Note that when using TensorFlow,
     for best performance you should set
@@ -126,27 +128,37 @@ def ResNet50(include_top=True, weights='imagenet',
             layers at the top of the network.
         weights: one of `None` (random initialization)
             or "imagenet" (pre-training on ImageNet).
-        input_tensor: optional Keras tensor (i.e. xput of `layers.Input()`)
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
             to use as image input for the model.
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(224, 224, 3)` (with `tf` dim ordering)
+            or `(3, 224, 244)` (with `th` dim ordering).
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 197.
+            E.g. `(200, 200, 3)` would be one valid value.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
 
     # Returns
         A Keras model instance.
-    '''
+    """
     if weights not in {'imagenet', None}:
         raise ValueError('The `weights` argument should be either '
                          '`None` (random initialization) or `imagenet` '
                          '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as imagenet with `include_top`'
+                         ' as true, `classes` should be 1000')
+
     # Determine proper input shape
-    if K.image_dim_ordering() == 'th':
-        if include_top:
-            input_shape = (3, 224, 224)
-        else:
-            input_shape = (3, None, None)
-    else:
-        if include_top:
-            input_shape = (224, 224, 3)
-        else:
-            input_shape = (None, None, 3)
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=224,
+                                      min_size=197,
+                                      dim_ordering=K.image_dim_ordering(),
+                                      include_top=include_top)
 
     if input_tensor is None:
         img_input = Input(shape=input_shape)
@@ -190,9 +202,16 @@ def ResNet50(include_top=True, weights='imagenet',
 
     if include_top:
         x = Flatten()(x)
-        x = Dense(1000, activation='softmax', name='fc1000')(x)
+        x = Dense(classes, activation='softmax', name='fc1000')(x)
 
-    model = Model(img_input, x)
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+    # Create model.
+    model = Model(inputs, x, name='resnet50')
 
     # load weights
     if weights == 'imagenet':

keras/applications/vgg16.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
-'''VGG16 model for Keras.
+"""VGG16 model for Keras.
 
-# Reference:
+# Reference
 
 - [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556)
 
-'''
+"""
 from __future__ import print_function
 from __future__ import absolute_import
 
@@ -14,10 +14,11 @@ import warnings
 from ..models import Model
 from ..layers import Flatten, Dense, Input
 from ..layers import Convolution2D, MaxPooling2D
+from ..engine.topology import get_source_inputs
 from ..utils.layer_utils import convert_all_kernels_in_model
 from ..utils.data_utils import get_file
 from .. import backend as K
-from .imagenet_utils import decode_predictions, preprocess_input
+from .imagenet_utils import decode_predictions, preprocess_input, _obtain_input_shape
 
 
 TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_th_dim_ordering_th_kernels.h5'
@@ -27,8 +28,9 @@ TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/relea
 
 
 def VGG16(include_top=True, weights='imagenet',
-          input_tensor=None):
-    '''Instantiate the VGG16 architecture,
+          input_tensor=None, input_shape=None,
+          classes=1000):
+    """Instantiate the VGG16 architecture,
     optionally loading weights pre-trained
     on ImageNet. Note that when using TensorFlow,
     for best performance you should set
@@ -47,25 +49,34 @@ def VGG16(include_top=True, weights='imagenet',
             or "imagenet" (pre-training on ImageNet).
         input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
             to use as image input for the model.
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(224, 224, 3)` (with `tf` dim ordering)
+            or `(3, 224, 244)` (with `th` dim ordering).
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 48.
+            E.g. `(200, 200, 3)` would be one valid value.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
 
     # Returns
         A Keras model instance.
-    '''
+    """
     if weights not in {'imagenet', None}:
         raise ValueError('The `weights` argument should be either '
                          '`None` (random initialization) or `imagenet` '
                          '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as imagenet with `include_top`'
+                         ' as true, `classes` should be 1000')
     # Determine proper input shape
-    if K.image_dim_ordering() == 'th':
-        if include_top:
-            input_shape = (3, 224, 224)
-        else:
-            input_shape = (3, None, None)
-    else:
-        if include_top:
-            input_shape = (224, 224, 3)
-        else:
-            input_shape = (None, None, 3)
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=224,
+                                      min_size=48,
+                                      dim_ordering=K.image_dim_ordering(),
+                                      include_top=include_top)
 
     if input_tensor is None:
         img_input = Input(shape=input_shape)
@@ -107,10 +118,16 @@ def VGG16(include_top=True, weights='imagenet',
         x = Flatten(name='flatten')(x)
         x = Dense(4096, activation='relu', name='fc1')(x)
         x = Dense(4096, activation='relu', name='fc2')(x)
-        x = Dense(1000, activation='softmax', name='predictions')(x)
+        x = Dense(classes, activation='softmax', name='predictions')(x)
 
-    # Create model
-    model = Model(img_input, x)
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+    # Create model.
+    model = Model(inputs, x, name='vgg16')
 
     # load weights
     if weights == 'imagenet':

keras/applications/vgg19.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
-'''VGG19 model for Keras.
+"""VGG19 model for Keras.
 
-# Reference:
+# Reference
 
 - [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556)
 
-'''
+"""
 from __future__ import print_function
 from __future__ import absolute_import
 
@@ -14,10 +14,11 @@ import warnings
 from ..models import Model
 from ..layers import Flatten, Dense, Input
 from ..layers import Convolution2D, MaxPooling2D
+from ..engine.topology import get_source_inputs
 from ..utils.layer_utils import convert_all_kernels_in_model
 from ..utils.data_utils import get_file
 from .. import backend as K
-from .imagenet_utils import decode_predictions, preprocess_input
+from .imagenet_utils import decode_predictions, preprocess_input, _obtain_input_shape
 
 
 TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_th_dim_ordering_th_kernels.h5'
@@ -27,8 +28,9 @@ TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/relea
 
 
 def VGG19(include_top=True, weights='imagenet',
-          input_tensor=None):
-    '''Instantiate the VGG19 architecture,
+          input_tensor=None, input_shape=None,
+          classes=1000):
+    """Instantiate the VGG19 architecture,
     optionally loading weights pre-trained
     on ImageNet. Note that when using TensorFlow,
     for best performance you should set
@@ -47,25 +49,34 @@ def VGG19(include_top=True, weights='imagenet',
             or "imagenet" (pre-training on ImageNet).
         input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
             to use as image input for the model.
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(224, 224, 3)` (with `tf` dim ordering)
+            or `(3, 224, 244)` (with `th` dim ordering).
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 48.
+            E.g. `(200, 200, 3)` would be one valid value.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
 
     # Returns
         A Keras model instance.
-    '''
+    """
     if weights not in {'imagenet', None}:
         raise ValueError('The `weights` argument should be either '
                          '`None` (random initialization) or `imagenet` '
                          '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as imagenet with `include_top`'
+                         ' as true, `classes` should be 1000')
     # Determine proper input shape
-    if K.image_dim_ordering() == 'th':
-        if include_top:
-            input_shape = (3, 224, 224)
-        else:
-            input_shape = (3, None, None)
-    else:
-        if include_top:
-            input_shape = (224, 224, 3)
-        else:
-            input_shape = (None, None, 3)
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=224,
+                                      min_size=48,
+                                      dim_ordering=K.image_dim_ordering(),
+                                      include_top=include_top)
 
     if input_tensor is None:
         img_input = Input(shape=input_shape)
@@ -110,10 +121,16 @@ def VGG19(include_top=True, weights='imagenet',
         x = Flatten(name='flatten')(x)
         x = Dense(4096, activation='relu', name='fc1')(x)
         x = Dense(4096, activation='relu', name='fc2')(x)
-        x = Dense(1000, activation='softmax', name='predictions')(x)
+        x = Dense(classes, activation='softmax', name='predictions')(x)
 
-    # Create model
-    model = Model(img_input, x)
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+    # Create model.
+    model = Model(inputs, x, name='vgg19')
 
     # load weights
     if weights == 'imagenet':

keras/applications/xception.py

@@ -0,0 +1,233 @@
+# -*- coding: utf-8 -*-
+"""Xception V1 model for Keras.
+
+On ImageNet, this model gets to a top-1 validation accuracy of 0.790
+and a top-5 validation accuracy of 0.945.
+
+Do note that the input image format for this model is different than for
+the VGG16 and ResNet models (299x299 instead of 224x224),
+and that the input preprocessing function
+is also different (same as Inception V3).
+
+Also do note that this model is only available for the TensorFlow backend,
+due to its reliance on `SeparableConvolution` layers.
+
+# Reference
+
+- [Xception: Deep Learning with Depthwise Separable Convolutions](https://arxiv.org/abs/1610.02357)
+
+"""
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..models import Model
+from ..layers import Dense, Input, BatchNormalization, Activation, merge
+from ..layers import Conv2D, SeparableConv2D, MaxPooling2D, GlobalAveragePooling2D
+from ..engine.topology import get_source_inputs
+from ..utils.data_utils import get_file
+from .. import backend as K
+from .imagenet_utils import decode_predictions, _obtain_input_shape
+
+
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def Xception(include_top=True, weights='imagenet',
+             input_tensor=None, input_shape=None,
+             classes=1000):
+    """Instantiate the Xception architecture,
+    optionally loading weights pre-trained
+    on ImageNet. This model is available for TensorFlow only,
+    and can only be used with inputs following the TensorFlow
+    dimension ordering `(width, height, channels)`.
+    You should set `image_dim_ordering="tf"` in your Keras config
+    located at ~/.keras/keras.json.
+
+    Note that the default input image size for this model is 299x299.
+
+    # Arguments
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(299, 299, 3)`.
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 71.
+            E.g. `(150, 150, 3)` would be one valid value.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+
+    # Returns
+        A Keras model instance.
+    """
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as imagenet with `include_top`'
+                         ' as true, `classes` should be 1000')
+
+    if K.backend() != 'tensorflow':
+        raise RuntimeError('The Xception model is only available with '
+                           'the TensorFlow backend.')
+    if K.image_dim_ordering() != 'tf':
+        warnings.warn('The Xception model is only available for the '
+                      'input dimension ordering "tf" '
+                      '(width, height, channels). '
+                      'However your settings specify the default '
+                      'dimension ordering "th" (channels, width, height). '
+                      'You should set `image_dim_ordering="tf"` in your Keras '
+                      'config located at ~/.keras/keras.json. '
+                      'The model being returned right now will expect inputs '
+                      'to follow the "tf" dimension ordering.')
+        K.set_image_dim_ordering('tf')
+        old_dim_ordering = 'th'
+    else:
+        old_dim_ordering = None
+
+    # Determine proper input shape
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=299,
+                                      min_size=71,
+                                      dim_ordering=K.image_dim_ordering(),
+                                      include_top=include_top)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+
+    x = Conv2D(32, 3, 3, subsample=(2, 2), bias=False, name='block1_conv1')(img_input)
+    x = BatchNormalization(name='block1_conv1_bn')(x)
+    x = Activation('relu', name='block1_conv1_act')(x)
+    x = Conv2D(64, 3, 3, bias=False, name='block1_conv2')(x)
+    x = BatchNormalization(name='block1_conv2_bn')(x)
+    x = Activation('relu', name='block1_conv2_act')(x)
+
+    residual = Conv2D(128, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = SeparableConv2D(128, 3, 3, border_mode='same', bias=False, name='block2_sepconv1')(x)
+    x = BatchNormalization(name='block2_sepconv1_bn')(x)
+    x = Activation('relu', name='block2_sepconv2_act')(x)
+    x = SeparableConv2D(128, 3, 3, border_mode='same', bias=False, name='block2_sepconv2')(x)
+    x = BatchNormalization(name='block2_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block2_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(256, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block3_sepconv1_act')(x)
+    x = SeparableConv2D(256, 3, 3, border_mode='same', bias=False, name='block3_sepconv1')(x)
+    x = BatchNormalization(name='block3_sepconv1_bn')(x)
+    x = Activation('relu', name='block3_sepconv2_act')(x)
+    x = SeparableConv2D(256, 3, 3, border_mode='same', bias=False, name='block3_sepconv2')(x)
+    x = BatchNormalization(name='block3_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block3_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(728, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block4_sepconv1_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block4_sepconv1')(x)
+    x = BatchNormalization(name='block4_sepconv1_bn')(x)
+    x = Activation('relu', name='block4_sepconv2_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block4_sepconv2')(x)
+    x = BatchNormalization(name='block4_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block4_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    for i in range(8):
+        residual = x
+        prefix = 'block' + str(i + 5)
+
+        x = Activation('relu', name=prefix + '_sepconv1_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv1')(x)
+        x = BatchNormalization(name=prefix + '_sepconv1_bn')(x)
+        x = Activation('relu', name=prefix + '_sepconv2_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv2')(x)
+        x = BatchNormalization(name=prefix + '_sepconv2_bn')(x)
+        x = Activation('relu', name=prefix + '_sepconv3_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv3')(x)
+        x = BatchNormalization(name=prefix + '_sepconv3_bn')(x)
+
+        x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(1024, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block13_sepconv1_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block13_sepconv1')(x)
+    x = BatchNormalization(name='block13_sepconv1_bn')(x)
+    x = Activation('relu', name='block13_sepconv2_act')(x)
+    x = SeparableConv2D(1024, 3, 3, border_mode='same', bias=False, name='block13_sepconv2')(x)
+    x = BatchNormalization(name='block13_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block13_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    x = SeparableConv2D(1536, 3, 3, border_mode='same', bias=False, name='block14_sepconv1')(x)
+    x = BatchNormalization(name='block14_sepconv1_bn')(x)
+    x = Activation('relu', name='block14_sepconv1_act')(x)
+
+    x = SeparableConv2D(2048, 3, 3, border_mode='same', bias=False, name='block14_sepconv2')(x)
+    x = BatchNormalization(name='block14_sepconv2_bn')(x)
+    x = Activation('relu', name='block14_sepconv2_act')(x)
+
+    if include_top:
+        x = GlobalAveragePooling2D(name='avg_pool')(x)
+        x = Dense(classes, activation='softmax', name='predictions')(x)
+
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+    # Create model.
+    model = Model(inputs, x, name='xception')
+
+    # load weights
+    if weights == 'imagenet':
+        if include_top:
+            weights_path = get_file('xception_weights_tf_dim_ordering_tf_kernels.h5',
+                                    TF_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        else:
+            weights_path = get_file('xception_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                    TF_WEIGHTS_PATH_NO_TOP,
+                                    cache_subdir='models')
+        model.load_weights(weights_path)
+
+    if old_dim_ordering:
+        K.set_image_dim_ordering(old_dim_ordering)
+    return model
+
+
+def preprocess_input(x):
+    x /= 255.
+    x -= 0.5
+    x *= 2.
+    return x

keras/backend/__init__.py

@@ -23,17 +23,20 @@ _keras_dir = os.path.join(_keras_base_dir, '.keras')
 if not os.path.exists(_keras_dir):
     os.makedirs(_keras_dir)
 
+# Default backend: TensorFlow.
 _BACKEND = 'tensorflow'
+
 _config_path = os.path.expanduser(os.path.join(_keras_dir, 'keras.json'))
 if os.path.exists(_config_path):
     _config = json.load(open(_config_path))
     _floatx = _config.get('floatx', floatx())
     assert _floatx in {'float16', 'float32', 'float64'}
     _epsilon = _config.get('epsilon', epsilon())
-    assert type(_epsilon) == float
+    assert isinstance(_epsilon, float)
     _backend = _config.get('backend', _BACKEND)
     assert _backend in {'theano', 'tensorflow'}
-    _image_dim_ordering = _config.get('image_dim_ordering', image_dim_ordering())
+    _image_dim_ordering = _config.get('image_dim_ordering',
+                                      image_dim_ordering())
     assert _image_dim_ordering in {'tf', 'th'}
 
     set_floatx(_floatx)
@@ -63,11 +66,11 @@ elif _BACKEND == 'tensorflow':
     sys.stderr.write('Using TensorFlow backend.\n')
     from .tensorflow_backend import *
 else:
-    raise Exception('Unknown backend: ' + str(_BACKEND))
+    raise ValueError('Unknown backend: ' + str(_BACKEND))
 
 
 def backend():
-    '''Publicly accessible method
+    """Publicly accessible method
     for determining the current backend.
-    '''
+    """
     return _BACKEND

keras/backend/common.py

@@ -11,58 +11,164 @@ _LEGACY_WEIGHT_ORDERING = False
 
 
 def epsilon():
-    '''Returns the value of the fuzz
+    """Returns the value of the fuzz
     factor used in numeric expressions.
-    '''
+
+    # Returns
+        A float.
+
+    # Example
+    ```python
+        >>> keras.backend.epsilon()
+        1e-08
+    ```
+    """
     return _EPSILON
 
 
 def set_epsilon(e):
-    '''Sets the value of the fuzz
+    """Sets the value of the fuzz
     factor used in numeric expressions.
-    '''
+
+    # Arguments
+        e: float. New value of epsilon.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> K.epsilon()
+        1e-08
+        >>> K.set_epsilon(1e-05)
+        >>> K.epsilon()
+        1e-05
+    ```
+    """
     global _EPSILON
     _EPSILON = e
 
 
 def floatx():
-    '''Returns the default float type, as a string
+    """Returns the default float type, as a string
     (e.g. 'float16', 'float32', 'float64').
-    '''
+
+    # Returns
+        String, the current default float type.
+
+    # Example
+    ```python
+        >>> keras.backend.floatx()
+        'float32'
+    ```
+    """
     return _FLOATX
 
 
 def set_floatx(floatx):
+    """Sets the default float type.
+
+    # Arguments
+        String: 'float16', 'float32', or 'float64'.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> K.floatx()
+        'float32'
+        >>> K.set_floatx('float16')
+        >>> K.floatx()
+        'float16'
+    ```
+    """
     global _FLOATX
     if floatx not in {'float16', 'float32', 'float64'}:
-        raise Exception('Unknown floatx type: ' + str(floatx))
+        raise ValueError('Unknown floatx type: ' + str(floatx))
     _FLOATX = str(floatx)
 
 
 def cast_to_floatx(x):
-    '''Cast a Numpy array to floatx.
-    '''
+    """Cast a Numpy array to the default Keras float type.
+
+    # Arguments
+        x: Numpy array.
+
+    # Returns
+        The same Numpy array, cast to its new type.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> K.floatx()
+        'float32'
+        >>> arr = numpy.array([1.0, 2.0], dtype='float64')
+        >>> arr.dtype
+        dtype('float64')
+        >>> new_arr = K.cast_to_floatx(arr)
+        >>> new_arr
+        array([ 1.,  2.], dtype=float32)
+        >>> new_arr.dtype
+        dtype('float32')
+    ```
+    """
     return np.asarray(x, dtype=_FLOATX)
 
 
 def image_dim_ordering():
-    '''Returns the image dimension ordering
+    """Returns the default image dimension ordering
     convention ('th' or 'tf').
-    '''
+
+    # Returns
+        A string, either `'th'` or `'tf'`
+
+    # Example
+    ```python
+        >>> keras.backend.image_dim_ordering()
+        'th'
+    ```
+    """
     return _IMAGE_DIM_ORDERING
 
 
 def set_image_dim_ordering(dim_ordering):
-    '''Sets the value of the image dimension
+    """Sets the value of the image dimension
     ordering convention ('th' or 'tf').
-    '''
+
+    # Arguments
+        dim_ordering: string. `'th'` or `'tf'`.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> K.image_dim_ordering()
+        'th'
+        >>> K.set_image_dim_ordering('tf')
+        >>> K.image_dim_ordering()
+        'tf'
+    ```
+    """
     global _IMAGE_DIM_ORDERING
     if dim_ordering not in {'tf', 'th'}:
-        raise Exception('Unknown dim_ordering:', dim_ordering)
+        raise ValueError('Unknown dim_ordering:', dim_ordering)
     _IMAGE_DIM_ORDERING = str(dim_ordering)
 
 
 def get_uid(prefix=''):
+    """Provides a unique UID given a string prefix.
+
+    # Arguments
+        prefix: string.
+
+    # Returns
+        An integer.
+
+    # Example
+    ```
+        >>> keras.backend.get_uid('dense')
+        >>> 1
+        >>> keras.backend.get_uid('dense')
+        >>> 2
+    ```
+
+    """
     _UID_PREFIXES[prefix] += 1
     return _UID_PREFIXES[prefix]
 
@@ -73,6 +179,28 @@ def reset_uids():
 
 
 def is_keras_tensor(x):
+    """Returns whether `x` is a Keras tensor.
+
+    # Arguments
+        x: a potential tensor.
+
+    # Returns
+        A boolean: whether the argument is a Keras tensor.
+
+    # Examples
+    ```python
+        >>> from keras import backend as K
+        >>> np_var = numpy.array([1, 2])
+        >>> K.is_keras_tensor(np_var)
+        False
+        >>> keras_var = K.variable(np_var)
+        >>> K.is_keras_tensor(keras_var)  # A variable is not a Tensor.
+        False
+        >>> keras_placeholder = K.placeholder(shape=(2, 4, 5))
+        >>> K.is_keras_tensor(keras_placeholder)  # A placeholder is a Tensor.
+        True
+    ```
+    """
     if hasattr(x, '_keras_shape'):
         return True
     else:

keras/callbacks.py

@@ -1,58 +1,109 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+import os
+import csv
+
 import numpy as np
 import time
 import json
 import warnings
 
 from collections import deque
+from collections import OrderedDict
+from collections import Iterable
 from .utils.generic_utils import Progbar
 from keras import backend as K
 from pkg_resources import parse_version
 
+try:
+    import requests
+except ImportError:
+    requests = None
+
+if K.backend() == 'tensorflow':
+    import tensorflow as tf
+
 
 class CallbackList(object):
-    def __init__(self, callbacks=[], queue_length=10):
+    """Container abstracting a list of callbacks.
+
+    # Arguments
+        callbacks: List of `Callback` instances.
+        queue_length: Queue length for keeping
+            running statistics over callback execution time.
+    """
+
+    def __init__(self, callbacks=None, queue_length=10):
+        callbacks = callbacks or []
         self.callbacks = [c for c in callbacks]
         self.queue_length = queue_length
 
     def append(self, callback):
         self.callbacks.append(callback)
 
-    def _set_params(self, params):
+    def set_params(self, params):
         for callback in self.callbacks:
-            callback._set_params(params)
+            callback.set_params(params)
 
-    def _set_model(self, model):
+    def set_model(self, model):
         for callback in self.callbacks:
-            callback._set_model(model)
+            callback.set_model(model)
 
-    def on_epoch_begin(self, epoch, logs={}):
+    def on_epoch_begin(self, epoch, logs=None):
+        """Called at the start of an epoch.
+
+        # Arguments
+            epoch: integer, index of epoch.
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
         for callback in self.callbacks:
             callback.on_epoch_begin(epoch, logs)
         self._delta_t_batch = 0.
         self._delta_ts_batch_begin = deque([], maxlen=self.queue_length)
         self._delta_ts_batch_end = deque([], maxlen=self.queue_length)
 
-    def on_epoch_end(self, epoch, logs={}):
+    def on_epoch_end(self, epoch, logs=None):
+        """Called at the end of an epoch.
+
+        # Arguments
+            epoch: integer, index of epoch.
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
         for callback in self.callbacks:
             callback.on_epoch_end(epoch, logs)
 
-    def on_batch_begin(self, batch, logs={}):
+    def on_batch_begin(self, batch, logs=None):
+        """Called right before processing a batch.
+
+        # Arguments
+            batch: integer, index of batch within the current epoch.
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
         t_before_callbacks = time.time()
         for callback in self.callbacks:
             callback.on_batch_begin(batch, logs)
         self._delta_ts_batch_begin.append(time.time() - t_before_callbacks)
         delta_t_median = np.median(self._delta_ts_batch_begin)
-        if self._delta_t_batch > 0. and delta_t_median > 0.95 * \
-           self._delta_t_batch and delta_t_median > 0.1:
+        if (self._delta_t_batch > 0. and
+           delta_t_median > 0.95 * self._delta_t_batch and
+           delta_t_median > 0.1):
             warnings.warn('Method on_batch_begin() is slow compared '
                           'to the batch update (%f). Check your callbacks.'
                           % delta_t_median)
         self._t_enter_batch = time.time()
 
-    def on_batch_end(self, batch, logs={}):
+    def on_batch_end(self, batch, logs=None):
+        """Called at the end of a batch.
+
+        # Arguments
+            batch: integer, index of batch within the current epoch.
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
         if not hasattr(self, '_t_enter_batch'):
             self._t_enter_batch = time.time()
         self._delta_t_batch = time.time() - self._t_enter_batch
@@ -61,22 +112,35 @@ class CallbackList(object):
             callback.on_batch_end(batch, logs)
         self._delta_ts_batch_end.append(time.time() - t_before_callbacks)
         delta_t_median = np.median(self._delta_ts_batch_end)
-        if self._delta_t_batch > 0. and (delta_t_median > 0.95 * self._delta_t_batch and delta_t_median > 0.1):
+        if (self._delta_t_batch > 0. and
+           (delta_t_median > 0.95 * self._delta_t_batch and delta_t_median > 0.1)):
             warnings.warn('Method on_batch_end() is slow compared '
                           'to the batch update (%f). Check your callbacks.'
                           % delta_t_median)
 
-    def on_train_begin(self, logs={}):
+    def on_train_begin(self, logs=None):
+        """Called at the beginning of training.
+
+        # Arguments
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
         for callback in self.callbacks:
             callback.on_train_begin(logs)
 
-    def on_train_end(self, logs={}):
+    def on_train_end(self, logs=None):
+        """Called at the end of training.
+
+        # Arguments
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
         for callback in self.callbacks:
             callback.on_train_end(logs)
 
 
 class Callback(object):
-    '''Abstract base class used to build new callbacks.
+    """Abstract base class used to build new callbacks.
 
     # Properties
         params: dict. Training parameters
@@ -100,47 +164,48 @@ class Callback(object):
             the number of samples in the current batch.
         on_batch_end: logs include `loss`, and optionally `acc`
             (if accuracy monitoring is enabled).
-    '''
+    """
+
     def __init__(self):
         pass
 
-    def _set_params(self, params):
+    def set_params(self, params):
         self.params = params
 
-    def _set_model(self, model):
+    def set_model(self, model):
         self.model = model
 
-    def on_epoch_begin(self, epoch, logs={}):
+    def on_epoch_begin(self, epoch, logs=None):
         pass
 
-    def on_epoch_end(self, epoch, logs={}):
+    def on_epoch_end(self, epoch, logs=None):
         pass
 
-    def on_batch_begin(self, batch, logs={}):
+    def on_batch_begin(self, batch, logs=None):
         pass
 
-    def on_batch_end(self, batch, logs={}):
+    def on_batch_end(self, batch, logs=None):
         pass
 
-    def on_train_begin(self, logs={}):
+    def on_train_begin(self, logs=None):
         pass
 
-    def on_train_end(self, logs={}):
+    def on_train_end(self, logs=None):
         pass
 
 
 class BaseLogger(Callback):
-    '''Callback that accumulates epoch averages of
-    the metrics being monitored.
+    """Callback that accumulates epoch averages of metrics.
 
-    This callback is automatically applied to
-    every Keras model.
-    '''
-    def on_epoch_begin(self, epoch, logs={}):
+    This callback is automatically applied to every Keras model.
+    """
+
+    def on_epoch_begin(self, epoch, logs=None):
         self.seen = 0
         self.totals = {}
 
-    def on_batch_end(self, batch, logs={}):
+    def on_batch_end(self, batch, logs=None):
+        logs = logs or {}
         batch_size = logs.get('size', 0)
         self.seen += batch_size
 
@@ -150,32 +215,35 @@ class BaseLogger(Callback):
             else:
                 self.totals[k] = v * batch_size
 
-    def on_epoch_end(self, epoch, logs={}):
-        for k in self.params['metrics']:
-            if k in self.totals:
-                # make value available to next callbacks
-                logs[k] = self.totals[k] / self.seen
+    def on_epoch_end(self, epoch, logs=None):
+        if logs is not None:
+            for k in self.params['metrics']:
+                if k in self.totals:
+                    # Make value available to next callbacks.
+                    logs[k] = self.totals[k] / self.seen
 
 
 class ProgbarLogger(Callback):
-    '''Callback that prints metrics to stdout.
-    '''
-    def on_train_begin(self, logs={}):
+    """Callback that prints metrics to stdout.
+    """
+
+    def on_train_begin(self, logs=None):
         self.verbose = self.params['verbose']
         self.nb_epoch = self.params['nb_epoch']
 
-    def on_epoch_begin(self, epoch, logs={}):
+    def on_epoch_begin(self, epoch, logs=None):
         if self.verbose:
             print('Epoch %d/%d' % (epoch + 1, self.nb_epoch))
             self.progbar = Progbar(target=self.params['nb_sample'],
                                    verbose=self.verbose)
         self.seen = 0
 
-    def on_batch_begin(self, batch, logs={}):
+    def on_batch_begin(self, batch, logs=None):
         if self.seen < self.params['nb_sample']:
             self.log_values = []
 
-    def on_batch_end(self, batch, logs={}):
+    def on_batch_end(self, batch, logs=None):
+        logs = logs or {}
         batch_size = logs.get('size', 0)
         self.seen += batch_size
 
@@ -183,12 +251,13 @@ class ProgbarLogger(Callback):
             if k in logs:
                 self.log_values.append((k, logs[k]))
 
-        # skip progbar update for the last batch;
-        # will be handled by on_epoch_end
+        # Skip progbar update for the last batch;
+        # will be handled by on_epoch_end.
         if self.verbose and self.seen < self.params['nb_sample']:
             self.progbar.update(self.seen, self.log_values)
 
-    def on_epoch_end(self, epoch, logs={}):
+    def on_epoch_end(self, epoch, logs=None):
+        logs = logs or {}
         for k in self.params['metrics']:
             if k in logs:
                 self.log_values.append((k, logs[k]))
@@ -197,33 +266,34 @@ class ProgbarLogger(Callback):
 
 
 class History(Callback):
-    '''Callback that records events
-    into a `History` object.
+    """Callback that records events into a `History` object.
 
     This callback is automatically applied to
     every Keras model. The `History` object
     gets returned by the `fit` method of models.
-    '''
-    def on_train_begin(self, logs={}):
+    """
+
+    def on_train_begin(self, logs=None):
         self.epoch = []
         self.history = {}
 
-    def on_epoch_end(self, epoch, logs={}):
+    def on_epoch_end(self, epoch, logs=None):
+        logs = logs or {}
         self.epoch.append(epoch)
         for k, v in logs.items():
             self.history.setdefault(k, []).append(v)
 
 
 class ModelCheckpoint(Callback):
-    '''Save the model after every epoch.
+    """Save the model after every epoch.
 
     `filepath` can contain named formatting options,
     which will be filled the value of `epoch` and
     keys in `logs` (passed in `on_epoch_end`).
 
     For example: if `filepath` is `weights.{epoch:02d}-{val_loss:.2f}.hdf5`,
-    then multiple files will be save with the epoch number and
-    the validation loss.
+    then the model checkpoints will be saved with the epoch number and
+    the validation loss in the filename.
 
     # Arguments
         filepath: string, path to save the model file.
@@ -243,17 +313,20 @@ class ModelCheckpoint(Callback):
         save_weights_only: if True, then only the model's weights will be
             saved (`model.save_weights(filepath)`), else the full model
             is saved (`model.save(filepath)`).
+        period: Interval (number of epochs) between checkpoints.
+    """
 
-    '''
     def __init__(self, filepath, monitor='val_loss', verbose=0,
                  save_best_only=False, save_weights_only=False,
-                 mode='auto'):
+                 mode='auto', period=1):
         super(ModelCheckpoint, self).__init__()
         self.monitor = monitor
         self.verbose = verbose
         self.filepath = filepath
         self.save_best_only = save_best_only
         self.save_weights_only = save_weights_only
+        self.period = period
+        self.epochs_since_last_save = 0
 
         if mode not in ['auto', 'min', 'max']:
             warnings.warn('ModelCheckpoint mode %s is unknown, '
@@ -268,50 +341,58 @@ class ModelCheckpoint(Callback):
             self.monitor_op = np.greater
             self.best = -np.Inf
         else:
-            if 'acc' in self.monitor:
+            if 'acc' in self.monitor or self.monitor.startswith('fmeasure'):
                 self.monitor_op = np.greater
                 self.best = -np.Inf
             else:
                 self.monitor_op = np.less
                 self.best = np.Inf
 
-    def on_epoch_end(self, epoch, logs={}):
-        filepath = self.filepath.format(epoch=epoch, **logs)
-        if self.save_best_only:
-            current = logs.get(self.monitor)
-            if current is None:
-                warnings.warn('Can save best model only with %s available, '
-                              'skipping.' % (self.monitor), RuntimeWarning)
-            else:
-                if self.monitor_op(current, self.best):
-                    if self.verbose > 0:
-                        print('Epoch %05d: %s improved from %0.5f to %0.5f,'
-                              ' saving model to %s'
-                              % (epoch, self.monitor, self.best,
-                                 current, filepath))
-                    self.best = current
-                    if self.save_weights_only:
-                        self.model.save_weights(filepath, overwrite=True)
-                    else:
-                        self.model.save(filepath, overwrite=True)
+    def on_epoch_end(self, epoch, logs=None):
+        logs = logs or {}
+        self.epochs_since_last_save += 1
+        if self.epochs_since_last_save >= self.period:
+            self.epochs_since_last_save = 0
+            filepath = self.filepath.format(epoch=epoch, **logs)
+            if self.save_best_only:
+                current = logs.get(self.monitor)
+                if current is None:
+                    warnings.warn('Can save best model only with %s available, '
+                                  'skipping.' % (self.monitor), RuntimeWarning)
                 else:
-                    if self.verbose > 0:
-                        print('Epoch %05d: %s did not improve' %
-                              (epoch, self.monitor))
-        else:
-            if self.verbose > 0:
-                print('Epoch %05d: saving model to %s' % (epoch, filepath))
-            if self.save_weights_only:
-                self.model.save_weights(filepath, overwrite=True)
+                    if self.monitor_op(current, self.best):
+                        if self.verbose > 0:
+                            print('Epoch %05d: %s improved from %0.5f to %0.5f,'
+                                  ' saving model to %s'
+                                  % (epoch, self.monitor, self.best,
+                                     current, filepath))
+                        self.best = current
+                        if self.save_weights_only:
+                            self.model.save_weights(filepath, overwrite=True)
+                        else:
+                            self.model.save(filepath, overwrite=True)
+                    else:
+                        if self.verbose > 0:
+                            print('Epoch %05d: %s did not improve' %
+                                  (epoch, self.monitor))
             else:
-                self.model.save(filepath, overwrite=True)
+                if self.verbose > 0:
+                    print('Epoch %05d: saving model to %s' % (epoch, filepath))
+                if self.save_weights_only:
+                    self.model.save_weights(filepath, overwrite=True)
+                else:
+                    self.model.save(filepath, overwrite=True)
 
 
 class EarlyStopping(Callback):
-    '''Stop training when a monitored quantity has stopped improving.
+    """Stop training when a monitored quantity has stopped improving.
 
     # Arguments
         monitor: quantity to be monitored.
+        min_delta: minimum change in the monitored quantity
+            to qualify as an improvement, i.e. an absolute
+            change of less than min_delta, will count as no
+            improvement.
         patience: number of epochs with no improvement
             after which training will be stopped.
         verbose: verbosity mode.
@@ -322,14 +403,18 @@ class EarlyStopping(Callback):
             monitored has stopped increasing; in `auto`
             mode, the direction is automatically inferred
             from the name of the monitored quantity.
-    '''
-    def __init__(self, monitor='val_loss', patience=0, verbose=0, mode='auto'):
+    """
+
+    def __init__(self, monitor='val_loss',
+                 min_delta=0, patience=0, verbose=0, mode='auto'):
         super(EarlyStopping, self).__init__()
 
         self.monitor = monitor
         self.patience = patience
         self.verbose = verbose
+        self.min_delta = min_delta
         self.wait = 0
+        self.stopped_epoch = 0
 
         if mode not in ['auto', 'min', 'max']:
             warnings.warn('EarlyStopping mode %s is unknown, '
@@ -342,90 +427,115 @@ class EarlyStopping(Callback):
         elif mode == 'max':
             self.monitor_op = np.greater
         else:
-            if 'acc' in self.monitor:
+            if 'acc' in self.monitor or self.monitor.startswith('fmeasure'):
                 self.monitor_op = np.greater
             else:
                 self.monitor_op = np.less
 
-    def on_train_begin(self, logs={}):
-        self.wait = 0       # Allow instances to be re-used
+        if self.monitor_op == np.greater:
+            self.min_delta *= 1
+        else:
+            self.min_delta *= -1
+
+    def on_train_begin(self, logs=None):
+        self.wait = 0  # Allow instances to be re-used
         self.best = np.Inf if self.monitor_op == np.less else -np.Inf
 
-    def on_epoch_end(self, epoch, logs={}):
+    def on_epoch_end(self, epoch, logs=None):
         current = logs.get(self.monitor)
         if current is None:
             warnings.warn('Early stopping requires %s available!' %
                           (self.monitor), RuntimeWarning)
 
-        if self.monitor_op(current, self.best):
+        if self.monitor_op(current - self.min_delta, self.best):
             self.best = current
             self.wait = 0
         else:
             if self.wait >= self.patience:
-                if self.verbose > 0:
-                    print('Epoch %05d: early stopping' % (epoch))
+                self.stopped_epoch = epoch
                 self.model.stop_training = True
             self.wait += 1
 
+    def on_train_end(self, logs=None):
+        if self.stopped_epoch > 0 and self.verbose > 0:
+            print('Epoch %05d: early stopping' % (self.stopped_epoch))
+
 
 class RemoteMonitor(Callback):
-    '''Callback used to stream events to a server.
+    """Callback used to stream events to a server.
 
     Requires the `requests` library.
+    Events are sent to `root + '/publish/epoch/end/'` by default. Calls are
+    HTTP POST, with a `data` argument which is a
+    JSON-encoded dictionary of event data.
 
     # Arguments
-        root: root url to which the events will be sent (at the end
-            of every epoch). Events are sent to
-            `root + '/publish/epoch/end/'` by default. Calls are
-            HTTP POST, with a `data` argument which is a
-            JSON-encoded dictionary of event data.
-    '''
+        root: String; root url of the target server.
+        path: String; path relative to `root` to which the events will be sent.
+        field: String; JSON field under which the data will be stored.
+        headers: Dictionary; optional custom HTTP headers.
+            Defaults to:
+            `{'Accept': 'application/json',
+              'Content-Type': 'application/json'}`
+    """
 
     def __init__(self,
                  root='http://localhost:9000',
                  path='/publish/epoch/end/',
-                 field='data'):
+                 field='data',
+                 headers=None):
         super(RemoteMonitor, self).__init__()
+        if headers is None:
+            headers = {'Accept': 'application/json',
+                       'Content-Type': 'application/json'}
         self.root = root
         self.path = path
         self.field = field
+        self.headers = headers
 
-    def on_epoch_end(self, epoch, logs={}):
-        import requests
+    def on_epoch_end(self, epoch, logs=None):
+        if requests is None:
+            raise ImportError('RemoteMonitor requires '
+                              'the `requests` library.')
+        logs = logs or {}
         send = {}
         send['epoch'] = epoch
         for k, v in logs.items():
             send[k] = v
         try:
             requests.post(self.root + self.path,
-                          {self.field: json.dumps(send)})
-        except:
-            print('Warning: could not reach RemoteMonitor '
-                  'root server at ' + str(self.root))
+                          {self.field: json.dumps(send)},
+                          headers=self.headers)
+        except requests.exceptions.RequestException:
+            warnings.warn('Warning: could not reach RemoteMonitor '
+                          'root server at ' + str(self.root))
 
 
 class LearningRateScheduler(Callback):
-    '''Learning rate scheduler.
+    """Learning rate scheduler.
 
     # Arguments
         schedule: a function that takes an epoch index as input
             (integer, indexed from 0) and returns a new
             learning rate as output (float).
-    '''
+    """
+
     def __init__(self, schedule):
         super(LearningRateScheduler, self).__init__()
         self.schedule = schedule
 
-    def on_epoch_begin(self, epoch, logs={}):
-        assert hasattr(self.model.optimizer, 'lr'), \
-            'Optimizer must have a "lr" attribute.'
+    def on_epoch_begin(self, epoch, logs=None):
+        if not hasattr(self.model.optimizer, 'lr'):
+            raise ValueError('Optimizer must have a "lr" attribute.')
         lr = self.schedule(epoch)
-        assert type(lr) == float, 'The output of the "schedule" function should be float.'
+        if not isinstance(lr, (float, np.float32, np.float64)):
+            raise ValueError('The output of the "schedule" function '
+                             'should be float.')
         K.set_value(self.model.optimizer.lr, lr)
 
 
 class TensorBoard(Callback):
-    ''' Tensorboard basic visualizations.
+    """Tensorboard basic visualizations.
 
     This callback writes a log for TensorBoard, which allows
     you to visualize dynamic graphs of your training and test
@@ -451,30 +561,33 @@ class TensorBoard(Callback):
         write_graph: whether to visualize the graph in Tensorboard.
             The log file can become quite large when
             write_graph is set to True.
-    '''
+    """
 
-    def __init__(self, log_dir='./logs', histogram_freq=0, write_graph=True, write_images=False):
+    def __init__(self, log_dir='./logs',
+                 histogram_freq=0,
+                 write_graph=True,
+                 write_images=False):
         super(TensorBoard, self).__init__()
-        if K._BACKEND != 'tensorflow':
-            raise Exception('TensorBoard callback only works '
-                            'with the TensorFlow backend.')
+        if K.backend() != 'tensorflow':
+            raise RuntimeError('TensorBoard callback only works '
+                               'with the TensorFlow backend.')
         self.log_dir = log_dir
         self.histogram_freq = histogram_freq
         self.merged = None
         self.write_graph = write_graph
         self.write_images = write_images
 
-    def _set_model(self, model):
-        import tensorflow as tf
-        import keras.backend.tensorflow_backend as KTF
-
+    def set_model(self, model):
         self.model = model
-        self.sess = KTF.get_session()
+        self.sess = K.get_session()
         if self.histogram_freq and self.merged is None:
             for layer in self.model.layers:
 
                 for weight in layer.weights:
-                    tf.histogram_summary(weight.name, weight)
+                    if hasattr(tf, 'histogram_summary'):
+                        tf.histogram_summary(weight.name, weight)
+                    else:
+                        tf.summary.histogram(weight.name, weight)
 
                     if self.write_images:
                         w_img = tf.squeeze(weight)
@@ -488,24 +601,42 @@ class TensorBoard(Callback):
 
                         w_img = tf.expand_dims(tf.expand_dims(w_img, 0), -1)
 
-                        tf.image_summary(weight.name, w_img)
+                        if hasattr(tf, 'image_summary'):
+                            tf.image_summary(weight.name, w_img)
+                        else:
+                            tf.summary.image(weight.name, w_img)
 
                 if hasattr(layer, 'output'):
-                    tf.histogram_summary('{}_out'.format(layer.name),
-                                         layer.output)
-        self.merged = tf.merge_all_summaries()
+                    if hasattr(tf, 'histogram_summary'):
+                        tf.histogram_summary('{}_out'.format(layer.name),
+                                             layer.output)
+                    else:
+                        tf.summary.histogram('{}_out'.format(layer.name),
+                                             layer.output)
+
+        if hasattr(tf, 'merge_all_summaries'):
+            self.merged = tf.merge_all_summaries()
+        else:
+            self.merged = tf.summary.merge_all()
+
         if self.write_graph:
-            if parse_version(tf.__version__) >= parse_version('0.8.0'):
+            if hasattr(tf, 'summary') and hasattr(tf.summary, 'FileWriter'):
+                self.writer = tf.summary.FileWriter(self.log_dir,
+                                                    self.sess.graph)
+            elif parse_version(tf.__version__) >= parse_version('0.8.0'):
                 self.writer = tf.train.SummaryWriter(self.log_dir,
                                                      self.sess.graph)
             else:
                 self.writer = tf.train.SummaryWriter(self.log_dir,
                                                      self.sess.graph_def)
         else:
-            self.writer = tf.train.SummaryWriter(self.log_dir)
+            if hasattr(tf, 'summary') and hasattr(tf.summary, 'FileWriter'):
+                self.writer = tf.summary.FileWriter(self.log_dir)
+            else:
+                self.writer = tf.train.SummaryWriter(self.log_dir)
 
-    def on_epoch_end(self, epoch, logs={}):
-        import tensorflow as tf
+    def on_epoch_end(self, epoch, logs=None):
+        logs = logs or {}
 
         if self.model.validation_data and self.histogram_freq:
             if epoch % self.histogram_freq == 0:
@@ -528,7 +659,272 @@ class TensorBoard(Callback):
                 continue
             summary = tf.Summary()
             summary_value = summary.value.add()
-            summary_value.simple_value = value
+            summary_value.simple_value = value.item()
             summary_value.tag = name
             self.writer.add_summary(summary, epoch)
         self.writer.flush()
+
+    def on_train_end(self, _):
+        self.writer.close()
+
+
+class ReduceLROnPlateau(Callback):
+    """Reduce learning rate when a metric has stopped improving.
+
+    Models often benefit from reducing the learning rate by a factor
+    of 2-10 once learning stagnates. This callback monitors a
+    quantity and if no improvement is seen for a 'patience' number
+    of epochs, the learning rate is reduced.
+
+    # Example
+        ```python
+            reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
+                                          patience=5, min_lr=0.001)
+            model.fit(X_train, Y_train, callbacks=[reduce_lr])
+        ```
+
+    # Arguments
+        monitor: quantity to be monitored.
+        factor: factor by which the learning rate will
+            be reduced. new_lr = lr * factor
+        patience: number of epochs with no improvement
+            after which learning rate will be reduced.
+        verbose: int. 0: quiet, 1: update messages.
+        mode: one of {auto, min, max}. In `min` mode,
+            lr will be reduced when the quantity
+            monitored has stopped decreasing; in `max`
+            mode it will be reduced when the quantity
+            monitored has stopped increasing; in `auto`
+            mode, the direction is automatically inferred
+            from the name of the monitored quantity.
+        epsilon: threshold for measuring the new optimum,
+            to only focus on significant changes.
+        cooldown: number of epochs to wait before resuming
+            normal operation after lr has been reduced.
+        min_lr: lower bound on the learning rate.
+    """
+
+    def __init__(self, monitor='val_loss', factor=0.1, patience=10,
+                 verbose=0, mode='auto', epsilon=1e-4, cooldown=0, min_lr=0):
+        super(ReduceLROnPlateau, self).__init__()
+
+        self.monitor = monitor
+        if factor >= 1.0:
+            raise ValueError('ReduceLROnPlateau '
+                             'does not support a factor >= 1.0.')
+        self.factor = factor
+        self.min_lr = min_lr
+        self.epsilon = epsilon
+        self.patience = patience
+        self.verbose = verbose
+        self.cooldown = cooldown
+        self.cooldown_counter = 0  # Cooldown counter.
+        self.wait = 0
+        self.best = 0
+        self.mode = mode
+        self.monitor_op = None
+        self._reset()
+
+    def _reset(self):
+        """Resets wait counter and cooldown counter.
+        """
+        if self.mode not in ['auto', 'min', 'max']:
+            warnings.warn('Learning Rate Plateau Reducing mode %s is unknown, '
+                          'fallback to auto mode.' % (self.mode),
+                          RuntimeWarning)
+            self.mode = 'auto'
+        if (self.mode == 'min' or
+           (self.mode == 'auto' and 'acc' not in self.monitor)):
+            self.monitor_op = lambda a, b: np.less(a, b - self.epsilon)
+            self.best = np.Inf
+        else:
+            self.monitor_op = lambda a, b: np.greater(a, b + self.epsilon)
+            self.best = -np.Inf
+        self.cooldown_counter = 0
+        self.wait = 0
+        self.lr_epsilon = self.min_lr * 1e-4
+
+    def on_train_begin(self, logs=None):
+        self._reset()
+
+    def on_epoch_end(self, epoch, logs=None):
+        logs = logs or {}
+        logs['lr'] = K.get_value(self.model.optimizer.lr)
+        current = logs.get(self.monitor)
+        if current is None:
+            warnings.warn('Learning Rate Plateau Reducing requires %s available!' %
+                          self.monitor, RuntimeWarning)
+        else:
+            if self.in_cooldown():
+                self.cooldown_counter -= 1
+                self.wait = 0
+
+            if self.monitor_op(current, self.best):
+                self.best = current
+                self.wait = 0
+            elif not self.in_cooldown():
+                if self.wait >= self.patience:
+                    old_lr = float(K.get_value(self.model.optimizer.lr))
+                    if old_lr > self.min_lr + self.lr_epsilon:
+                        new_lr = old_lr * self.factor
+                        new_lr = max(new_lr, self.min_lr)
+                        K.set_value(self.model.optimizer.lr, new_lr)
+                        if self.verbose > 0:
+                            print('\nEpoch %05d: reducing learning rate to %s.' % (epoch, new_lr))
+                        self.cooldown_counter = self.cooldown
+                        self.wait = 0
+                self.wait += 1
+
+    def in_cooldown(self):
+        return self.cooldown_counter > 0
+
+
+class CSVLogger(Callback):
+    """Callback that streams epoch results to a csv file.
+
+    Supports all values that can be represented as a string,
+    including 1D iterables such as np.ndarray.
+
+    # Example
+        ```python
+            csv_logger = CSVLogger('training.log')
+            model.fit(X_train, Y_train, callbacks=[csv_logger])
+        ```
+
+    # Arguments
+        filename: filename of the csv file, e.g. 'run/log.csv'.
+        separator: string used to separate elements in the csv file.
+        append: True: append if file exists (useful for continuing
+            training). False: overwrite existing file,
+    """
+
+    def __init__(self, filename, separator=',', append=False):
+        self.sep = separator
+        self.filename = filename
+        self.append = append
+        self.writer = None
+        self.keys = None
+        self.append_header = True
+        super(CSVLogger, self).__init__()
+
+    def on_train_begin(self, logs=None):
+        if self.append:
+            if os.path.exists(self.filename):
+                with open(self.filename) as f:
+                    self.append_header = not bool(len(f.readline()))
+            self.csv_file = open(self.filename, 'a')
+        else:
+            self.csv_file = open(self.filename, 'w')
+
+    def on_epoch_end(self, epoch, logs=None):
+        logs = logs or {}
+
+        def handle_value(k):
+            is_zero_dim_ndarray = isinstance(k, np.ndarray) and k.ndim == 0
+            if isinstance(k, Iterable) and not is_zero_dim_ndarray:
+                return '"[%s]"' % (', '.join(map(str, k)))
+            else:
+                return k
+
+        if not self.writer:
+            self.keys = sorted(logs.keys())
+
+            class CustomDialect(csv.excel):
+                delimiter = self.sep
+
+            self.writer = csv.DictWriter(self.csv_file,
+                                         fieldnames=['epoch'] + self.keys, dialect=CustomDialect)
+            if self.append_header:
+                self.writer.writeheader()
+
+        row_dict = OrderedDict({'epoch': epoch})
+        row_dict.update((key, handle_value(logs[key])) for key in self.keys)
+        self.writer.writerow(row_dict)
+        self.csv_file.flush()
+
+    def on_train_end(self, logs=None):
+        self.csv_file.close()
+        self.writer = None
+
+
+class LambdaCallback(Callback):
+    """Callback for creating simple, custom callbacks on-the-fly.
+
+    This callback is constructed with anonymous functions that will be called
+    at the appropriate time. Note that the callbacks expects positional
+    arguments, as:
+     - `on_epoch_begin` and `on_epoch_end` expect two positional arguments:
+        `epoch`, `logs`
+     - `on_batch_begin` and `on_batch_end` expect two positional arguments:
+        `batch`, `logs`
+     - `on_train_begin` and `on_train_end` expect one positional argument:
+        `logs`
+
+    # Arguments
+        on_epoch_begin: called at the beginning of every epoch.
+        on_epoch_end: called at the end of every epoch.
+        on_batch_begin: called at the beginning of every batch.
+        on_batch_end: called at the end of every batch.
+        on_train_begin: called at the beginning of model training.
+        on_train_end: called at the end of model training.
+
+    # Example
+        ```python
+        # Print the batch number at the beginning of every batch.
+        batch_print_callback = LambdaCallback(
+            on_batch_begin=lambda batch,logs: print(batch))
+
+        # Plot the loss after every epoch.
+        import numpy as np
+        import matplotlib.pyplot as plt
+        plot_loss_callback = LambdaCallback(
+            on_epoch_end=lambda epoch, logs: plt.plot(np.arange(epoch),
+                                                      logs['loss']))
+
+        # Terminate some processes after having finished model training.
+        processes = ...
+        cleanup_callback = LambdaCallback(
+            on_train_end=lambda logs: [
+                p.terminate() for p in processes if p.is_alive()])
+
+        model.fit(...,
+                  callbacks=[batch_print_callback,
+                             plot_loss_callback,
+                             cleanup_callback])
+        ```
+    """
+
+    def __init__(self,
+                 on_epoch_begin=None,
+                 on_epoch_end=None,
+                 on_batch_begin=None,
+                 on_batch_end=None,
+                 on_train_begin=None,
+                 on_train_end=None,
+                 **kwargs):
+        super(LambdaCallback, self).__init__()
+        self.__dict__.update(kwargs)
+        if on_epoch_begin is not None:
+            self.on_epoch_begin = on_epoch_begin
+        else:
+            self.on_epoch_begin = lambda epoch, logs: None
+        if on_epoch_end is not None:
+            self.on_epoch_end = on_epoch_end
+        else:
+            self.on_epoch_end = lambda epoch, logs: None
+        if on_batch_begin is not None:
+            self.on_batch_begin = on_batch_begin
+        else:
+            self.on_batch_begin = lambda batch, logs: None
+        if on_batch_end is not None:
+            self.on_batch_end = on_batch_end
+        else:
+            self.on_batch_end = lambda batch, logs: None
+        if on_train_begin is not None:
+            self.on_train_begin = on_train_begin
+        else:
+            self.on_train_begin = lambda logs: None
+        if on_train_end is not None:
+            self.on_train_end = on_train_end
+        else:
+            self.on_train_end = lambda logs: None

keras/constraints.py

@@ -1,8 +1,10 @@
 from __future__ import absolute_import
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 class Constraint(object):
+
     def __call__(self, p):
         return p
 
@@ -11,26 +13,29 @@ class Constraint(object):
 
 
 class MaxNorm(Constraint):
-    '''Constrain the weights incident to each hidden unit to have a norm less than or equal to a desired value.
+    """MaxNorm weight constraint.
+
+    Constrains the weights incident to each hidden unit
+    to have a norm less than or equal to a desired value.
 
     # Arguments
         m: the maximum norm for the incoming weights.
-        axis: integer, axis along which to calculate weight norms. For instance,
-            in a `Dense` layer the weight matrix has shape (input_dim, output_dim),
-            set `axis` to `0` to constrain each weight vector of length (input_dim).
-            In a `MaxoutDense` layer the weight tensor has shape (nb_feature, input_dim, output_dim),
-            set `axis` to `1` to constrain each weight vector of length (input_dim),
-            i.e. constrain the filters incident to the `max` operation.
-            In a `Convolution2D` layer with the Theano backend, the weight tensor
-            has shape (nb_filter, stack_size, nb_row, nb_col), set `axis` to `[1,2,3]`
-            to constrain the weights of each filter tensor of size (stack_size, nb_row, nb_col).
-            In a `Convolution2D` layer with the TensorFlow backend, the weight tensor
-            has shape (nb_row, nb_col, stack_size, nb_filter), set `axis` to `[0,1,2]`
-            to constrain the weights of each filter tensor of size (nb_row, nb_col, stack_size).
+        axis: integer, axis along which to calculate weight norms.
+            For instance, in a `Dense` layer the weight matrix
+            has shape `(input_dim, output_dim)`,
+            set `axis` to `0` to constrain each weight vector
+            of length `(input_dim,)`.
+            In a `Convolution2D` layer with `dim_ordering="tf"`,
+            the weight tensor has shape
+            `(rows, cols, input_depth, output_depth)`,
+            set `axis` to `[0, 1, 2]`
+            to constrain the weights of each filter tensor of size
+            `(rows, cols, input_depth)`.
 
     # References
         - [Dropout: A Simple Way to Prevent Neural Networks from Overfitting Srivastava, Hinton, et al. 2014](http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf)
-    '''
+    """
+
     def __init__(self, m=2, axis=0):
         self.m = m
         self.axis = axis
@@ -38,7 +43,7 @@ class MaxNorm(Constraint):
     def __call__(self, p):
         norms = K.sqrt(K.sum(K.square(p), axis=self.axis, keepdims=True))
         desired = K.clip(norms, 0, self.m)
-        p = p * (desired / (K.epsilon() + norms))
+        p *= (desired / (K.epsilon() + norms))
         return p
 
     def get_config(self):
@@ -48,46 +53,98 @@ class MaxNorm(Constraint):
 
 
 class NonNeg(Constraint):
-    '''Constrain the weights to be non-negative.
-    '''
+    """Constrains the weights to be non-negative.
+    """
+
     def __call__(self, p):
         p *= K.cast(p >= 0., K.floatx())
         return p
 
 
 class UnitNorm(Constraint):
-    '''Constrain the weights incident to each hidden unit to have unit norm.
+    """Constrains the weights incident to each hidden unit to have unit norm.
 
     # Arguments
-        axis: integer, axis along which to calculate weight norms. For instance,
-            in a `Dense` layer the weight matrix has shape (input_dim, output_dim),
-            set `axis` to `0` to constrain each weight vector of length (input_dim).
-            In a `MaxoutDense` layer the weight tensor has shape (nb_feature, input_dim, output_dim),
-            set `axis` to `1` to constrain each weight vector of length (input_dim),
-            i.e. constrain the filters incident to the `max` operation.
-            In a `Convolution2D` layer with the Theano backend, the weight tensor
-            has shape (nb_filter, stack_size, nb_row, nb_col), set `axis` to `[1,2,3]`
-            to constrain the weights of each filter tensor of size (stack_size, nb_row, nb_col).
-            In a `Convolution2D` layer with the TensorFlow backend, the weight tensor
-            has shape (nb_row, nb_col, stack_size, nb_filter), set `axis` to `[0,1,2]`
-            to constrain the weights of each filter tensor of size (nb_row, nb_col, stack_size).
-    '''
+        axis: integer, axis along which to calculate weight norms.
+            For instance, in a `Dense` layer the weight matrix
+            has shape `(input_dim, output_dim)`,
+            set `axis` to `0` to constrain each weight vector
+            of length `(input_dim,)`.
+            In a `Convolution2D` layer with `dim_ordering="tf"`,
+            the weight tensor has shape
+            `(rows, cols, input_depth, output_depth)`,
+            set `axis` to `[0, 1, 2]`
+            to constrain the weights of each filter tensor of size
+            `(rows, cols, input_depth)`.
+    """
+
     def __init__(self, axis=0):
         self.axis = axis
 
     def __call__(self, p):
-        return p / (K.epsilon() + K.sqrt(K.sum(K.square(p), axis=self.axis, keepdims=True)))
+        return p / (K.epsilon() + K.sqrt(K.sum(K.square(p),
+                                               axis=self.axis,
+                                               keepdims=True)))
 
     def get_config(self):
         return {'name': self.__class__.__name__,
                 'axis': self.axis}
 
 
+class MinMaxNorm(Constraint):
+    """MinMaxNorm weight constraint.
+
+    Constrains the weights incident to each hidden unit
+    to have the norm between a lower bound and an upper bound.
+
+    # Arguments
+        low: the minimum norm for the incoming weights.
+        high: the maximum norm for the incoming weights.
+        rate: rate for enforcing the constraint: weights will be
+            rescaled to yield (1 - rate) * norm + rate * norm.clip(low, high).
+            Effectively, this means that rate=1.0 stands for strict
+            enforcement of the constraint, while rate<1.0 means that
+            weights will be rescaled at each step to slowly move
+            towards a value inside the desired interval.
+        axis: integer, axis along which to calculate weight norms.
+            For instance, in a `Dense` layer the weight matrix
+            has shape `(input_dim, output_dim)`,
+            set `axis` to `0` to constrain each weight vector
+            of length `(input_dim,)`.
+            In a `Convolution2D` layer with `dim_ordering="tf"`,
+            the weight tensor has shape
+            `(rows, cols, input_depth, output_depth)`,
+            set `axis` to `[0, 1, 2]`
+            to constrain the weights of each filter tensor of size
+            `(rows, cols, input_depth)`.
+    """
+    def __init__(self, low=0.0, high=1.0, rate=1.0, axis=0):
+        self.low = low
+        self.high = high
+        self.rate = rate
+        self.axis = axis
+
+    def __call__(self, p):
+        norms = K.sqrt(K.sum(K.square(p), axis=self.axis, keepdims=True))
+        desired = self.rate * K.clip(norms, self.low, self.high) + (1 - self.rate) * norms
+        p *= (desired / (K.epsilon() + norms))
+        return p
+
+    def get_config(self):
+        return {'name': self.__class__.__name__,
+                'low': self.low,
+                'high': self.high,
+                'rate': self.rate,
+                'axis': self.axis}
+
+
+# Aliases.
+
 maxnorm = MaxNorm
 nonneg = NonNeg
 unitnorm = UnitNorm
 
-from .utils.generic_utils import get_from_module
+
 def get(identifier, kwargs=None):
     return get_from_module(identifier, globals(), 'constraint',
                            instantiate=True, kwargs=kwargs)

keras/datasets/cifar.py

@@ -5,17 +5,28 @@ from six.moves import cPickle
 
 
 def load_batch(fpath, label_key='labels'):
+    """Internal utility for parsing CIFAR data.
+
+    # Arguments
+        fpath: path the file to parse.
+        label_key: key for label data in the retrieve
+            dictionary.
+
+    # Returns
+        A tuple `(data, labels)`.
+    """
     f = open(fpath, 'rb')
     if sys.version_info < (3,):
         d = cPickle.load(f)
     else:
-        d = cPickle.load(f, encoding="bytes")
+        d = cPickle.load(f, encoding='bytes')
         # decode utf8
+        d_decoded = {}
         for k, v in d.items():
-            del(d[k])
-            d[k.decode("utf8")] = v
+            d_decoded[k.decode('utf8')] = v
+        d = d_decoded
     f.close()
-    data = d["data"]
+    data = d['data']
     labels = d[label_key]
 
     data = data.reshape(data.shape[0], 3, 32, 32)

keras/datasets/cifar10.py

@@ -7,29 +7,34 @@ import os
 
 
 def load_data():
-    dirname = "cifar-10-batches-py"
-    origin = "http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
+    """Loads CIFAR10 dataset.
+
+    # Returns
+        Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
+    """
+    dirname = 'cifar-10-batches-py'
+    origin = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
     path = get_file(dirname, origin=origin, untar=True)
 
     nb_train_samples = 50000
 
-    X_train = np.zeros((nb_train_samples, 3, 32, 32), dtype="uint8")
-    y_train = np.zeros((nb_train_samples,), dtype="uint8")
+    x_train = np.zeros((nb_train_samples, 3, 32, 32), dtype='uint8')
+    y_train = np.zeros((nb_train_samples,), dtype='uint8')
 
     for i in range(1, 6):
         fpath = os.path.join(path, 'data_batch_' + str(i))
         data, labels = load_batch(fpath)
-        X_train[(i - 1) * 10000: i * 10000, :, :, :] = data
+        x_train[(i - 1) * 10000: i * 10000, :, :, :] = data
         y_train[(i - 1) * 10000: i * 10000] = labels
 
     fpath = os.path.join(path, 'test_batch')
-    X_test, y_test = load_batch(fpath)
+    x_test, y_test = load_batch(fpath)
 
     y_train = np.reshape(y_train, (len(y_train), 1))
     y_test = np.reshape(y_test, (len(y_test), 1))
 
     if K.image_dim_ordering() == 'tf':
-        X_train = X_train.transpose(0, 2, 3, 1)
-        X_test = X_test.transpose(0, 2, 3, 1)
+        x_train = x_train.transpose(0, 2, 3, 1)
+        x_test = x_test.transpose(0, 2, 3, 1)
 
-    return (X_train, y_train), (X_test, y_test)
+    return (x_train, y_train), (x_test, y_test)

keras/datasets/cifar100.py

@@ -7,24 +7,35 @@ import os
 
 
 def load_data(label_mode='fine'):
-    if label_mode not in ['fine', 'coarse']:
-        raise Exception('label_mode must be one of "fine" "coarse".')
+    """Loads CIFAR100 dataset.
 
-    dirname = "cifar-100-python"
-    origin = "http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
+    # Arguments
+        label_mode: one of "fine", "coarse".
+
+    # Returns
+        Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
+
+    # Raises
+        ValueError: in case of invalid `label_mode`.
+    """
+    if label_mode not in ['fine', 'coarse']:
+        raise ValueError('label_mode must be one of "fine" "coarse".')
+
+    dirname = 'cifar-100-python'
+    origin = 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
     path = get_file(dirname, origin=origin, untar=True)
 
     fpath = os.path.join(path, 'train')
-    X_train, y_train = load_batch(fpath, label_key=label_mode+'_labels')
+    x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
 
     fpath = os.path.join(path, 'test')
-    X_test, y_test = load_batch(fpath, label_key=label_mode+'_labels')
+    x_test, y_test = load_batch(fpath, label_key=label_mode + '_labels')
 
     y_train = np.reshape(y_train, (len(y_train), 1))
     y_test = np.reshape(y_test, (len(y_test), 1))
 
     if K.image_dim_ordering() == 'tf':
-        X_train = X_train.transpose(0, 2, 3, 1)
-        X_test = X_test.transpose(0, 2, 3, 1)
+        x_train = x_train.transpose(0, 2, 3, 1)
+        x_test = x_test.transpose(0, 2, 3, 1)
 
-    return (X_train, y_train), (X_test, y_test)
+    return (x_train, y_train), (x_test, y_test)

keras/datasets/data_utils.py

@@ -1,4 +0,0 @@
-from ..utils.data_utils import *
-import warnings
-
-warnings.warn('data_utils has been moved to keras.utils.data_utils.')

keras/datasets/imdb.py

@@ -10,9 +10,10 @@ import sys
 def load_data(path='imdb_full.pkl', nb_words=None, skip_top=0,
               maxlen=None, seed=113,
               start_char=1, oov_char=2, index_from=3):
-    '''
+    """Loads the IMDB dataset.
+
     # Arguments
-        path: where to store the data (in `/.keras/dataset`)
+        path: where to cache the data (relative to `~/.keras/dataset`).
         nb_words: max number of words to include. Words are ranked
             by how often they occur (in the training set) and only
             the most frequent words are kept
@@ -26,12 +27,19 @@ def load_data(path='imdb_full.pkl', nb_words=None, skip_top=0,
             or `skip_top` limit will be replaced with this character.
         index_from: index actual words with this index and higher.
 
+    # Returns
+        Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
+
+    # Raises
+        ValueError: in case `maxlen` is so low
+            that no input sequence could be kept.
+
     Note that the 'out of vocabulary' character is only used for
     words that were present in the training set but are not included
     because they're not making the `nb_words` cut here.
     Words that were not seen in the trining set but are in the test set
     have simply been skipped.
-    '''
+    """
     path = get_file(path,
                     origin='https://s3.amazonaws.com/text-datasets/imdb_full.pkl',
                     md5_hash='d091312047c43cf9e4e38fef92437263')
@@ -54,54 +62,63 @@ def load_data(path='imdb_full.pkl', nb_words=None, skip_top=0,
     np.random.seed(seed * 2)
     np.random.shuffle(labels_test)
 
-    X = x_train + x_test
+    xs = x_train + x_test
     labels = labels_train + labels_test
 
     if start_char is not None:
-        X = [[start_char] + [w + index_from for w in x] for x in X]
+        xs = [[start_char] + [w + index_from for w in x] for x in xs]
     elif index_from:
-        X = [[w + index_from for w in x] for x in X]
+        xs = [[w + index_from for w in x] for x in xs]
 
     if maxlen:
-        new_X = []
+        new_xs = []
         new_labels = []
-        for x, y in zip(X, labels):
+        for x, y in zip(xs, labels):
             if len(x) < maxlen:
-                new_X.append(x)
+                new_xs.append(x)
                 new_labels.append(y)
-        X = new_X
+        xs = new_xs
         labels = new_labels
-    if not X:
-        raise Exception('After filtering for sequences shorter than maxlen=' +
-                        str(maxlen) + ', no sequence was kept. '
-                        'Increase maxlen.')
+    if not xs:
+        raise ValueError('After filtering for sequences shorter than maxlen=' +
+                         str(maxlen) + ', no sequence was kept. '
+                         'Increase maxlen.')
     if not nb_words:
-        nb_words = max([max(x) for x in X])
+        nb_words = max([max(x) for x in xs])
 
     # by convention, use 2 as OOV word
-    # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
+    # reserve 'index_from' (=3 by default) characters:
+    # 0 (padding), 1 (start), 2 (OOV)
     if oov_char is not None:
-        X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
+        xs = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in xs]
     else:
-        nX = []
-        for x in X:
+        new_xs = []
+        for x in xs:
             nx = []
             for w in x:
-                if (w >= nb_words or w < skip_top):
+                if w >= nb_words or w < skip_top:
                     nx.append(w)
-            nX.append(nx)
-        X = nX
+            new_xs.append(nx)
+        xs = new_xs
 
-    X_train = np.array(X[:len(x_train)])
+    x_train = np.array(xs[:len(x_train)])
     y_train = np.array(labels[:len(x_train)])
 
-    X_test = np.array(X[len(x_train):])
+    x_test = np.array(xs[len(x_train):])
     y_test = np.array(labels[len(x_train):])
 
-    return (X_train, y_train), (X_test, y_test)
+    return (x_train, y_train), (x_test, y_test)
 
 
 def get_word_index(path='imdb_word_index.pkl'):
+    """Retrieves the dictionary mapping word indices back to words.
+
+    # Arguments
+        path: where to cache the data (relative to `~/.keras/dataset`).
+
+    # Returns
+        The word index dictionary.
+    """
     path = get_file(path,
                     origin='https://s3.amazonaws.com/text-datasets/imdb_word_index.pkl',
                     md5_hash='72d94b01291be4ff843198d3b0e1e4d7')

keras/datasets/mnist.py

@@ -1,14 +1,22 @@
-# -*- coding: utf-8 -*-
 import gzip
 from ..utils.data_utils import get_file
 from six.moves import cPickle
 import sys
 
 
-def load_data(path="mnist.pkl.gz"):
-    path = get_file(path, origin="https://s3.amazonaws.com/img-datasets/mnist.pkl.gz")
+def load_data(path='mnist.pkl.gz'):
+    """Loads the MNIST dataset.
 
-    if path.endswith(".gz"):
+    # Arguments
+        path: path where to cache the dataset locally
+            (relative to ~/.keras/datasets).
+
+    # Returns
+        Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
+    """
+    path = get_file(path, origin='https://s3.amazonaws.com/img-datasets/mnist.pkl.gz')
+
+    if path.endswith('.gz'):
         f = gzip.open(path, 'rb')
     else:
         f = open(path, 'rb')
@@ -16,7 +24,7 @@ def load_data(path="mnist.pkl.gz"):
     if sys.version_info < (3,):
         data = cPickle.load(f)
     else:
-        data = cPickle.load(f, encoding="bytes")
+        data = cPickle.load(f, encoding='bytes')
 
     f.close()
-    return data  # (X_train, y_train), (X_test, y_test)
+    return data  # (x_train, y_train), (x_test, y_test)

keras/datasets/reuters.py

@@ -10,9 +10,10 @@ import sys
 def load_data(path='reuters.pkl', nb_words=None, skip_top=0,
               maxlen=None, test_split=0.2, seed=113,
               start_char=1, oov_char=2, index_from=3):
-    '''
+    """Loads the Reuters newswire classification dataset.
+
     # Arguments
-        path: where to store the data (in `/.keras/dataset`)
+        path: where to cache the data (relative to `~/.keras/dataset`).
         nb_words: max number of words to include. Words are ranked
             by how often they occur (in the training set) and only
             the most frequent words are kept
@@ -27,65 +28,77 @@ def load_data(path='reuters.pkl', nb_words=None, skip_top=0,
             or `skip_top` limit will be replaced with this character.
         index_from: index actual words with this index and higher.
 
+    # Returns
+        Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
+
     Note that the 'out of vocabulary' character is only used for
     words that were present in the training set but are not included
     because they're not making the `nb_words` cut here.
     Words that were not seen in the trining set but are in the test set
     have simply been skipped.
-    '''
+    """
 
     path = get_file(path, origin='https://s3.amazonaws.com/text-datasets/reuters.pkl')
     f = open(path, 'rb')
-    X, labels = cPickle.load(f)
+    xs, labels = cPickle.load(f)
     f.close()
 
     np.random.seed(seed)
-    np.random.shuffle(X)
+    np.random.shuffle(xs)
     np.random.seed(seed)
     np.random.shuffle(labels)
 
     if start_char is not None:
-        X = [[start_char] + [w + index_from for w in x] for x in X]
+        xs = [[start_char] + [w + index_from for w in x] for x in xs]
     elif index_from:
-        X = [[w + index_from for w in x] for x in X]
+        xs = [[w + index_from for w in x] for x in xs]
 
     if maxlen:
-        new_X = []
+        new_xs = []
         new_labels = []
-        for x, y in zip(X, labels):
+        for x, y in zip(xs, labels):
             if len(x) < maxlen:
-                new_X.append(x)
+                new_xs.append(x)
                 new_labels.append(y)
-        X = new_X
+        xs = new_xs
         labels = new_labels
 
     if not nb_words:
-        nb_words = max([max(x) for x in X])
+        nb_words = max([max(x) for x in xs])
 
     # by convention, use 2 as OOV word
-    # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
+    # reserve 'index_from' (=3 by default) characters:
+    # 0 (padding), 1 (start), 2 (OOV)
     if oov_char is not None:
-        X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
+        xs = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in xs]
     else:
-        nX = []
-        for x in X:
+        new_xs = []
+        for x in xs:
             nx = []
             for w in x:
-                if (w >= nb_words or w < skip_top):
+                if w >= nb_words or w < skip_top:
                     nx.append(w)
-            nX.append(nx)
-        X = nX
+            new_xs.append(nx)
+        xs = new_xs
 
-    X_train = X[:int(len(X) * (1 - test_split))]
-    y_train = labels[:int(len(X) * (1 - test_split))]
+    x_train = xs[:int(len(xs) * (1 - test_split))]
+    y_train = labels[:int(len(xs) * (1 - test_split))]
 
-    X_test = X[int(len(X) * (1 - test_split)):]
-    y_test = labels[int(len(X) * (1 - test_split)):]
+    x_test = xs[int(len(xs) * (1 - test_split)):]
+    y_test = labels[int(len(xs) * (1 - test_split)):]
 
-    return (X_train, y_train), (X_test, y_test)
+    return (x_train, y_train), (x_test, y_test)
 
 
 def get_word_index(path='reuters_word_index.pkl'):
+    """Retrieves the dictionary mapping word indices back to words.
+
+    # Arguments
+        path: where to cache the data (relative to `~/.keras/dataset`).
+
+    # Returns
+        The word index dictionary.
+    """
     path = get_file(path, origin='https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl')
     f = open(path, 'rb')
 

keras/initializations.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import numpy as np
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def get_fans(shape, dim_ordering='th'):
@@ -8,7 +9,7 @@ def get_fans(shape, dim_ordering='th'):
         fan_in = shape[0]
         fan_out = shape[1]
     elif len(shape) == 4 or len(shape) == 5:
-        # assuming convolution kernels (2D or 3D).
+        # Assuming convolution kernels (2D or 3D).
         # TH kernel shape: (depth, input_depth, ...)
         # TF kernel shape: (..., input_depth, depth)
         if dim_ordering == 'th':
@@ -20,34 +21,40 @@ def get_fans(shape, dim_ordering='th'):
             fan_in = shape[-2] * receptive_field_size
             fan_out = shape[-1] * receptive_field_size
         else:
-            raise Exception('Invalid dim_ordering: ' + dim_ordering)
+            raise ValueError('Invalid dim_ordering: ' + dim_ordering)
     else:
-        # no specific assumptions
+        # No specific assumptions.
         fan_in = np.sqrt(np.prod(shape))
         fan_out = np.sqrt(np.prod(shape))
     return fan_in, fan_out
 
 
-def uniform(shape, scale=0.05, name=None):
+def uniform(shape, scale=0.05, name=None, dim_ordering='th'):
     return K.random_uniform_variable(shape, -scale, scale, name=name)
 
 
-def normal(shape, scale=0.05, name=None):
+def normal(shape, scale=0.05, name=None, dim_ordering='th'):
     return K.random_normal_variable(shape, 0.0, scale, name=name)
 
 
 def lecun_uniform(shape, name=None, dim_ordering='th'):
-    ''' Reference: LeCun 98, Efficient Backprop
+    """LeCun uniform variance scaling initializer.
+
+    # References
+        LeCun 98, Efficient Backprop,
         http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf
-    '''
+    """
     fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
     scale = np.sqrt(3. / fan_in)
     return uniform(shape, scale, name=name)
 
 
 def glorot_normal(shape, name=None, dim_ordering='th'):
-    ''' Reference: Glorot & Bengio, AISTATS 2010
-    '''
+    """Glorot normal variance scaling initializer.
+
+    # References
+        Glorot & Bengio, AISTATS 2010
+    """
     fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
     s = np.sqrt(2. / (fan_in + fan_out))
     return normal(shape, s, name=name)
@@ -60,48 +67,55 @@ def glorot_uniform(shape, name=None, dim_ordering='th'):
 
 
 def he_normal(shape, name=None, dim_ordering='th'):
-    ''' Reference:  He et al., http://arxiv.org/abs/1502.01852
-    '''
+    """He normal variance scaling initializer.
+
+    # References
+        He et al., http://arxiv.org/abs/1502.01852
+    """
     fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
     s = np.sqrt(2. / fan_in)
     return normal(shape, s, name=name)
 
 
 def he_uniform(shape, name=None, dim_ordering='th'):
+    """He uniform variance scaling initializer.
+    """
     fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
     s = np.sqrt(6. / fan_in)
     return uniform(shape, s, name=name)
 
 
-def orthogonal(shape, scale=1.1, name=None):
-    ''' From Lasagne. Reference: Saxe et al., http://arxiv.org/abs/1312.6120
-    '''
+def orthogonal(shape, scale=1.1, name=None, dim_ordering='th'):
+    """Orthogonal initializer.
+
+    # References
+        Saxe et al., http://arxiv.org/abs/1312.6120
+    """
     flat_shape = (shape[0], np.prod(shape[1:]))
     a = np.random.normal(0.0, 1.0, flat_shape)
     u, _, v = np.linalg.svd(a, full_matrices=False)
-    # pick the one with the correct shape
+    # Pick the one with the correct shape.
     q = u if u.shape == flat_shape else v
     q = q.reshape(shape)
     return K.variable(scale * q[:shape[0], :shape[1]], name=name)
 
 
-def identity(shape, scale=1, name=None):
+def identity(shape, scale=1, name=None, dim_ordering='th'):
     if len(shape) != 2 or shape[0] != shape[1]:
-        raise Exception('Identity matrix initialization can only be used '
-                        'for 2D square matrices.')
+        raise ValueError('Identity matrix initialization can only be used '
+                         'for 2D square matrices.')
     else:
         return K.variable(scale * np.identity(shape[0]), name=name)
 
 
-def zero(shape, name=None):
+def zero(shape, name=None, dim_ordering='th'):
     return K.zeros(shape, name=name)
 
 
-def one(shape, name=None):
+def one(shape, name=None, dim_ordering='th'):
     return K.ones(shape, name=name)
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier, **kwargs):
     return get_from_module(identifier, globals(),
                            'initialization', kwargs=kwargs)

keras/layers/__init__.py

@@ -10,3 +10,4 @@ from .embeddings import *
 from .noise import *
 from .advanced_activations import *
 from .wrappers import *
+from .convolutional_recurrent import *

keras/layers/advanced_activations.py

@@ -5,8 +5,9 @@ import numpy as np
 
 
 class LeakyReLU(Layer):
-    '''Special version of a Rectified Linear Unit
-    that allows a small gradient when the unit is not active:
+    """Leaky version of a Rectified Linear Unit.
+
+    It allows a small gradient when the unit is not active:
     `f(x) = alpha * x for x < 0`,
     `f(x) = x for x >= 0`.
 
@@ -20,7 +21,11 @@ class LeakyReLU(Layer):
 
     # Arguments
         alpha: float >= 0. Negative slope coefficient.
-    '''
+
+    # References
+        - [Rectifier Nonlinearities Improve Neural Network Acoustic Models](https://web.stanford.edu/~awni/papers/relu_hybrid_icml2013_final.pdf)
+    """
+
     def __init__(self, alpha=0.3, **kwargs):
         self.supports_masking = True
         self.alpha = alpha
@@ -36,7 +41,9 @@ class LeakyReLU(Layer):
 
 
 class PReLU(Layer):
-    '''Parametric Rectified Linear Unit:
+    """Parametric Rectified Linear Unit.
+
+    It follows:
     `f(x) = alphas * x for x < 0`,
     `f(x) = x for x >= 0`,
     where `alphas` is a learned array with the same shape as x.
@@ -52,18 +59,38 @@ class PReLU(Layer):
     # Arguments
         init: initialization function for the weights.
         weights: initial weights, as a list of a single Numpy array.
+        shared_axes: the axes along which to share learnable
+            parameters for the activation function.
+            For example, if the incoming feature maps
+            are from a 2D convolution
+            with output shape `(batch, height, width, channels)`,
+            and you wish to share parameters across space
+            so that each filter only has one set of parameters,
+            set `shared_axes=[1, 2]`.
 
     # References
-        - [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](http://arxiv.org/pdf/1502.01852v1.pdf)
-    '''
-    def __init__(self, init='zero', weights=None, **kwargs):
+        - [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](https://arxiv.org/abs/1502.01852)
+    """
+
+    def __init__(self, init='zero', weights=None, shared_axes=None, **kwargs):
         self.supports_masking = True
         self.init = initializations.get(init)
         self.initial_weights = weights
+        if not isinstance(shared_axes, (list, tuple)):
+            self.shared_axes = [shared_axes]
+        else:
+            self.shared_axes = list(shared_axes)
         super(PReLU, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        self.alphas = self.init(input_shape[1:],
+        param_shape = list(input_shape[1:])
+        self.param_broadcast = [False] * len(param_shape)
+        if self.shared_axes[0] is not None:
+            for i in self.shared_axes:
+                param_shape[i - 1] = 1
+                self.param_broadcast[i - 1] = True
+
+        self.alphas = self.init(param_shape,
                                 name='{}_alphas'.format(self.name))
         self.trainable_weights = [self.alphas]
 
@@ -73,7 +100,11 @@ class PReLU(Layer):
 
     def call(self, x, mask=None):
         pos = K.relu(x)
-        neg = self.alphas * (x - abs(x)) * 0.5
+        if K.backend() == 'theano':
+            neg = (K.pattern_broadcast(self.alphas, self.param_broadcast) *
+                   (x - K.abs(x)) * 0.5)
+        else:
+            neg = self.alphas * (x - K.abs(x)) * 0.5
         return pos + neg
 
     def get_config(self):
@@ -83,7 +114,9 @@ class PReLU(Layer):
 
 
 class ELU(Layer):
-    '''Exponential Linear Unit:
+    """Exponential Linear Unit.
+
+    It follows:
     `f(x) =  alpha * (exp(x) - 1.) for x < 0`,
     `f(x) = x for x >= 0`.
 
@@ -99,17 +132,16 @@ class ELU(Layer):
         alpha: scale for the negative factor.
 
     # References
-        - [Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)](http://arxiv.org/pdf/1511.07289v1.pdf)
-    '''
+        - [Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)](https://arxiv.org/abs/1511.07289v1)
+    """
+
     def __init__(self, alpha=1.0, **kwargs):
         self.supports_masking = True
         self.alpha = K.cast_to_floatx(alpha)
         super(ELU, self).__init__(**kwargs)
 
     def call(self, x, mask=None):
-        pos = K.relu(x)
-        neg = (x - abs(x)) * 0.5
-        return pos + self.alpha * (K.exp(neg) - 1.)
+        return K.elu(x, self.alpha)
 
     def get_config(self):
         config = {'alpha': float(self.alpha)}
@@ -118,8 +150,10 @@ class ELU(Layer):
 
 
 class ParametricSoftplus(Layer):
-    '''Parametric Softplus:
-    `alpha * log(1 + exp(beta * x))`
+    """Parametric Softplus.
+
+    It follows:
+    `f(x) = alpha * log(1 + exp(beta * x))`
 
     # Input shape
         Arbitrary. Use the keyword argument `input_shape`
@@ -133,23 +167,42 @@ class ParametricSoftplus(Layer):
         alpha_init: float. Initial value of the alpha weights.
         beta_init: float. Initial values of the beta weights.
         weights: initial weights, as a list of 2 numpy arrays.
+        shared_axes: the axes along which to share learnable
+            parameters for the activation function.
+            For example, if the incoming feature maps
+            are from a 2D convolution
+            with output shape `(batch, height, width, channels)`,
+            and you wish to share parameters across space
+            so that each filter only has one set of parameters,
+            set `shared_axes=[1, 2]`.
 
     # References
         - [Inferring Nonlinear Neuronal Computation Based on Physiologically Plausible Inputs](http://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1003143)
-    '''
+    """
+
     def __init__(self, alpha_init=0.2, beta_init=5.0,
-                 weights=None, **kwargs):
+                 weights=None, shared_axes=None, **kwargs):
         self.supports_masking = True
         self.alpha_init = K.cast_to_floatx(alpha_init)
         self.beta_init = K.cast_to_floatx(beta_init)
         self.initial_weights = weights
+        if not isinstance(shared_axes, (list, tuple)):
+            self.shared_axes = [shared_axes]
+        else:
+            self.shared_axes = list(shared_axes)
         super(ParametricSoftplus, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        input_shape = input_shape[1:]
-        self.alphas = K.variable(self.alpha_init * np.ones(input_shape),
+        param_shape = list(input_shape[1:])
+        self.param_broadcast = [False] * len(param_shape)
+        if self.shared_axes[0] is not None:
+            for i in self.shared_axes:
+                param_shape[i - 1] = 1
+                self.param_broadcast[i - 1] = True
+
+        self.alphas = K.variable(self.alpha_init * np.ones(param_shape),
                                  name='{}_alphas'.format(self.name))
-        self.betas = K.variable(self.beta_init * np.ones(input_shape),
+        self.betas = K.variable(self.beta_init * np.ones(param_shape),
                                 name='{}_betas'.format(self.name))
         self.trainable_weights = [self.alphas, self.betas]
 
@@ -158,7 +211,12 @@ class ParametricSoftplus(Layer):
             del self.initial_weights
 
     def call(self, x, mask=None):
-        return K.softplus(self.betas * x) * self.alphas
+        if K.backend() == 'theano':
+            return (K.softplus(K.pattern_broadcast(self.betas,
+                                                   self.param_broadcast) * x) *
+                    K.pattern_broadcast(self.alphas, self.param_broadcast))
+        else:
+            return K.softplus(self.betas * x) * self.alphas
 
     def get_config(self):
         config = {'alpha_init': float(self.alpha_init),
@@ -168,8 +226,10 @@ class ParametricSoftplus(Layer):
 
 
 class ThresholdedReLU(Layer):
-    '''Thresholded Rectified Linear Unit:
-    `f(x) = x for x > theta`
+    """Thresholded Rectified Linear Unit.
+
+    It follows:
+    `f(x) = x for x > theta`,
     `f(x) = 0 otherwise`.
 
     # Input shape
@@ -184,8 +244,9 @@ class ThresholdedReLU(Layer):
         theta: float >= 0. Threshold location of activation.
 
     # References
-        - [Zero-Bias Autoencoders and the Benefits of Co-Adapting Features](http://arxiv.org/pdf/1402.3337.pdf)
-    '''
+        - [Zero-Bias Autoencoders and the Benefits of Co-Adapting Features](http://arxiv.org/abs/1402.3337)
+    """
+
     def __init__(self, theta=1.0, **kwargs):
         self.supports_masking = True
         self.theta = K.cast_to_floatx(theta)
@@ -201,7 +262,12 @@ class ThresholdedReLU(Layer):
 
 
 class SReLU(Layer):
-    '''S-shaped Rectified Linear Unit.
+    """S-shaped Rectified Linear Unit.
+
+    It follows:
+    `f(x) = t^r + a^r(x - t^r) for x >= t^r`,
+    `f(x) = x for t^r > x > t^l`,
+    `f(x) = t^l + a^l(x - t^l) for x <= t^l`.
 
     # Input shape
         Arbitrary. Use the keyword argument `input_shape`
@@ -216,46 +282,77 @@ class SReLU(Layer):
         a_left_init: initialization function for the left part slope
         t_right_init: initialization function for the right part intercept
         a_right_init: initialization function for the right part slope
+        shared_axes: the axes along which to share learnable
+            parameters for the activation function.
+            For example, if the incoming feature maps
+            are from a 2D convolution
+            with output shape `(batch, height, width, channels)`,
+            and you wish to share parameters across space
+            so that each filter only has one set of parameters,
+            set `shared_axes=[1, 2]`.
 
     # References
         - [Deep Learning with S-shaped Rectified Linear Activation Units](http://arxiv.org/abs/1512.07030)
-    '''
+    """
+
     def __init__(self, t_left_init='zero', a_left_init='glorot_uniform',
-                 t_right_init='glorot_uniform', a_right_init='one', **kwargs):
+                 t_right_init='glorot_uniform', a_right_init='one',
+                 shared_axes=None, **kwargs):
         self.supports_masking = True
         self.t_left_init = t_left_init
         self.a_left_init = a_left_init
         self.t_right_init = t_right_init
         self.a_right_init = a_right_init
+        if not isinstance(shared_axes, (list, tuple)):
+            self.shared_axes = [shared_axes]
+        else:
+            self.shared_axes = list(shared_axes)
         super(SReLU, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        input_shape = input_shape[1:]
+        param_shape = list(input_shape[1:])
+        self.param_broadcast = [False] * len(param_shape)
+        if self.shared_axes[0] is not None:
+            for i in self.shared_axes:
+                param_shape[i - 1] = 1
+                self.param_broadcast[i - 1] = True
 
         t_left_init = initializations.get(self.t_left_init)
         a_left_init = initializations.get(self.a_left_init)
         t_right_init = initializations.get(self.t_right_init)
         a_right_init = initializations.get(self.a_right_init)
 
-        self.t_left = t_left_init(input_shape,
+        self.t_left = t_left_init(param_shape,
                                   name='{}_t_left'.format(self.name))
-        self.a_left = a_left_init(input_shape,
+        self.a_left = a_left_init(param_shape,
                                   name='{}_a_left'.format(self.name))
-        self.t_right = t_right_init(input_shape,
+        self.t_right = t_right_init(param_shape,
                                     name='{}_t_right'.format(self.name))
-        self.a_right = a_right_init(input_shape,
+        self.a_right = a_right_init(param_shape,
                                     name='{}_a_right'.format(self.name))
         # ensure the the right part is always to the right of the left
-        self.t_right_actual = self.t_left + abs(self.t_right)
+        self.t_right_actual = self.t_left + K.abs(self.t_right)
         self.trainable_weights = [self.t_left, self.a_left,
                                   self.t_right, self.a_right]
 
     def call(self, x, mask=None):
-        Y_left_and_center = self.t_left + K.relu(x - self.t_left,
-                                                 self.a_left,
-                                                 self.t_right_actual - self.t_left)
-        Y_right = K.relu(x - self.t_right_actual) * self.a_right
-        return Y_left_and_center + Y_right
+        if K.backend() == 'theano':
+            t_left = K.pattern_broadcast(self.t_left, self.param_broadcast)
+            a_left = K.pattern_broadcast(self.a_left, self.param_broadcast)
+            a_right = K.pattern_broadcast(self.a_right, self.param_broadcast)
+            t_right_actual = K.pattern_broadcast(self.t_right_actual,
+                                                 self.param_broadcast)
+        else:
+            t_left = self.t_left
+            a_left = self.a_left
+            a_right = self.a_right
+            t_right_actual = self.t_right_actual
+
+        y_left_and_center = t_left + K.relu(x - t_left,
+                                            a_left,
+                                            t_right_actual - t_left)
+        y_right = K.relu(x - t_right_actual) * a_right
+        return y_left_and_center + y_right
 
     def get_config(self):
         config = {'t_left_init': self.t_left_init,

keras/layers/convolutional_recurrent.py

@@ -0,0 +1,515 @@
+from .. import backend as K
+from .. import activations
+from .. import initializations
+from .. import regularizers
+
+import numpy as np
+from ..engine import Layer
+from ..engine import InputSpec
+from ..utils.np_utils import conv_output_length
+import warnings
+
+
+class ConvRecurrent2D(Layer):
+    """Abstract base class for convolutional recurrent layers.
+
+    Do not use in a model -- it's not a functional layer!
+
+    ConvLSTM2D
+    follow the specifications of this class and accept
+    the keyword arguments listed below.
+
+    # Input shape
+        5D tensor with shape `(nb_samples, timesteps, channels, rows, cols)`.
+
+    # Output shape
+        - if `return_sequences`: 5D tensor with shape
+            `(nb_samples, timesteps, channels, rows, cols)`.
+        - else, 4D tensor with shape `(nb_samples, channels, rows, cols)`.
+
+    # Arguments
+        weights: list of numpy arrays to set as initial weights.
+            The list should have 3 elements, of shapes:
+            `[(input_dim, nb_filter), (nb_filter, nb_filter), (nb_filter,)]`.
+        return_sequences: Boolean. Whether to return the last output
+            in the output sequence, or the full sequence.
+        go_backwards: Boolean (default False).
+            If True, rocess the input sequence backwards.
+        stateful: Boolean (default False). If True, the last state
+            for each sample at index i in a batch will be used as initial
+            state for the sample of index i in the following batch.
+        nb_filter: Number of convolution filters to use.
+        nb_row: Number of rows in the convolution kernel.
+        nb_col: Number of columns in the convolution kernel.
+            is required when using this layer as the first layer in a model.
+        input_shape: input_shape
+
+    # Masking
+        This layer supports masking for input data with a variable number
+        of timesteps. To introduce masks to your data,
+        use an [Embedding](embeddings.md) layer with the `mask_zero` parameter
+        set to `True`.
+        **Note:** for the time being, masking is only supported with Theano.
+
+    # TensorFlow warning
+        For the time being, when using the TensorFlow backend,
+        the number of timesteps used must be specified in your model.
+        Make sure to pass an `input_length` int argument to your
+        recurrent layer (if it comes first in your model),
+        or to pass a complete `input_shape` argument to the first layer
+        in your model otherwise.
+
+
+    # Note on using statefulness in RNNs
+        You can set RNN layers to be 'stateful', which means that the states
+        computed for the samples in one batch will be reused as initial states
+        for the samples in the next batch.
+        This assumes a one-to-one mapping between
+        samples in different successive batches.
+
+        To enable statefulness:
+            - specify `stateful=True` in the layer constructor.
+            - specify a fixed batch size for your model, by passing
+                a `batch_input_size=(...)` to the first layer in your model.
+                This is the expected shape of your inputs *including the batch
+                size*.
+                It should be a tuple of integers, e.g. `(32, 10, 100)`.
+
+        To reset the states of your model, call `.reset_states()` on either
+        a specific layer, or on your entire model.
+    """
+
+    def __init__(self, weights=None, nb_row=None, nb_col=None, nb_filter=None,
+                 return_sequences=False, go_backwards=False, stateful=False,
+                 dim_ordering=None, **kwargs):
+        self.return_sequences = return_sequences
+        self.go_backwards = go_backwards
+        self.stateful = stateful
+        self.initial_weights = weights
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.nb_filter = nb_filter
+        self.dim_ordering = dim_ordering
+        self.input_spec = [InputSpec(ndim=5)]
+
+        super(ConvRecurrent2D, self).__init__(**kwargs)
+
+    def compute_mask(self, input, mask):
+        if self.return_sequences:
+            return mask
+        else:
+            return None
+
+    def get_output_shape_for(self, input_shape):
+
+        if self.dim_ordering == 'th':
+            rows = input_shape[3]
+            cols = input_shape[4]
+        elif self.dim_ordering == 'tf':
+            rows = input_shape[2]
+            cols = input_shape[3]
+        else:
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+
+        rows = conv_output_length(rows, self.nb_row,
+                                  self.border_mode, self.subsample[0])
+        cols = conv_output_length(cols, self.nb_col,
+                                  self.border_mode, self.subsample[1])
+
+        if self.return_sequences:
+            if self.dim_ordering == 'th':
+                return (input_shape[0], input_shape[1],
+                        self.nb_filter, rows, cols)
+            elif self.dim_ordering == 'tf':
+                return (input_shape[0], input_shape[1],
+                        rows, cols, self.nb_filter)
+        else:
+            if self.dim_ordering == 'th':
+                return (input_shape[0], self.nb_filter, rows, cols)
+            elif self.dim_ordering == 'tf':
+                return (input_shape[0], rows, cols, self.nb_filter)
+
+    def step(self, x, states):
+        raise NotImplementedError
+
+    def get_constants(self, X, train=False):
+        return None
+
+    def get_initial_states(self, X):
+        # (samples, timesteps, row, col, filter)
+        initial_state = K.zeros_like(X)
+        # (samples,row, col, filter)
+        initial_state = K.sum(initial_state, axis=1)
+        initial_state = self.conv_step(initial_state, K.zeros(self.W_shape),
+                                       border_mode=self.border_mode)
+
+        initial_states = [initial_state for _ in range(2)]
+        return initial_states
+
+    def preprocess_input(self, x):
+        return x
+
+    def call(self, x, mask=None):
+        assert K.ndim(x) == 5
+        input_shape = self.input_spec[0].shape
+        unroll = False
+
+        if self.stateful:
+            initial_states = self.states
+        else:
+            initial_states = self.get_initial_states(x)
+
+        constants = self.get_constants(x)
+        preprocessed_input = self.preprocess_input(x)
+
+        last_output, outputs, states = K.rnn(self.step, preprocessed_input,
+                                             initial_states,
+                                             go_backwards=self.go_backwards,
+                                             mask=mask,
+                                             constants=constants,
+                                             unroll=unroll,
+                                             input_length=input_shape[1])
+        if self.stateful:
+            self.updates = []
+            for i in range(len(states)):
+                self.updates.append((self.states[i], states[i]))
+
+        if self.return_sequences:
+            return outputs
+        else:
+            return last_output
+
+    def get_config(self):
+        config = {'return_sequences': self.return_sequences,
+                  'go_backwards': self.go_backwards,
+                  'stateful': self.stateful}
+        if self.stateful:
+            config['batch_input_shape'] = self.input_spec[0].shape
+
+        base_config = super(ConvRecurrent2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class ConvLSTM2D(ConvRecurrent2D):
+    """Convolutional LSTM.
+
+    # Input shape
+        - if dim_ordering='th'
+            5D tensor with shape:
+            `(samples,time, channels, rows, cols)`
+        - if dim_ordering='tf'
+            5D tensor with shape:
+            `(samples,time, rows, cols, channels)`
+
+     # Output shape
+        - if `return_sequences`
+             - if dim_ordering='th'
+                5D tensor with shape:
+                `(samples, time, nb_filter, output_row, output_col)`
+             - if dim_ordering='tf'
+                5D tensor with shape:
+                `(samples, time, output_row, output_col, nb_filter)`
+        - else
+            - if dim_ordering ='th'
+                4D tensor with shape:
+                `(samples, nb_filter, output_row, output_col)`
+            - if dim_ordering='tf'
+                4D tensor with shape:
+                `(samples, output_row, output_col, nb_filter)`
+
+        where o_row and o_col depend on the shape of the filter and
+        the border_mode
+
+        # Arguments
+            nb_filter: Number of convolution filters to use.
+            nb_row: Number of rows in the convolution kernel.
+            nb_col: Number of columns in the convolution kernel.
+            border_mode: 'valid' or 'same'.
+            subsample: tuple of length 2. Factor by which to subsample output.
+                Also called strides elsewhere.
+            dim_ordering: 'tf' if the feature are at the last dimension or 'th'
+            stateful : Boolean (default False). If True, the last state
+                for each sample at index i in a batch will be used as initial
+                state for the sample of index i in the following batch.
+            init: weight initialization function.
+                Can be the name of an existing function (str),
+                or a Theano function
+                (see: [initializations](../initializations.md)).
+            inner_init: initialization function of the inner cells.
+            forget_bias_init: initialization function for the bias of the
+            forget gate.
+                [Jozefowicz et al.](http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf)
+                recommend initializing with ones.
+            activation: activation function.
+                Can be the name of an existing function (str),
+                or a Theano function (see: [activations](../activations.md)).
+            inner_activation: activation function for the inner cells.
+
+    # References
+        - [Convolutional LSTM Network: A Machine Learning Approach for
+        Precipitation Nowcasting](http://arxiv.org/abs/1506.04214v1)
+        The current implementation does not include the feedback loop on the
+        cells output
+    """
+
+    def __init__(self, nb_filter, nb_row, nb_col,
+                 init='glorot_uniform', inner_init='orthogonal',
+                 forget_bias_init='one', activation='tanh',
+                 inner_activation='hard_sigmoid',
+                 dim_ordering='default',
+                 border_mode='valid', subsample=(1, 1),
+                 W_regularizer=None, U_regularizer=None, b_regularizer=None,
+                 dropout_W=0., dropout_U=0., **kwargs):
+
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        if dim_ordering not in {'tf', 'th'}:
+            raise ValueError('dim_ordering must be in {tf,th}', dim_ordering)
+        self.nb_filter = nb_filter
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.init = initializations.get(init)
+        self.inner_init = initializations.get(inner_init)
+        self.forget_bias_init = initializations.get(forget_bias_init)
+        self.activation = activations.get(activation)
+        self.inner_activation = activations.get(inner_activation)
+        self.border_mode = border_mode
+        self.subsample = subsample
+
+        if dim_ordering == 'th':
+            warnings.warn('Be carefull if used with convolution3D layers:\n'
+                          'th in convolution 3D corresponds to '
+                          '(samples, channels, conv_dim1, conv_dim2,'
+                          'conv_dim3)\n'
+                          'while for this network it corresponds to: '
+                          '(samples, time, channels, rows, cols)')
+        self.dim_ordering = dim_ordering
+
+        kwargs['nb_filter'] = nb_filter
+        kwargs['nb_row'] = nb_row
+        kwargs['nb_col'] = nb_col
+        kwargs['dim_ordering'] = dim_ordering
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.U_regularizer = regularizers.get(U_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.dropout_W, self.dropout_U = dropout_W, dropout_U
+        if self.dropout_W or self.dropout_U:
+            self.uses_learning_phase = True
+
+        super(ConvLSTM2D, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        self.input_spec = [InputSpec(shape=input_shape)]
+
+        if self.dim_ordering == 'th':
+            stack_size = input_shape[2]
+            self.W_shape = (self.nb_filter, stack_size,
+                            self.nb_row, self.nb_col)
+        elif self.dim_ordering == 'tf':
+            stack_size = input_shape[4]
+            self.W_shape = (self.nb_row, self.nb_col,
+                            stack_size, self.nb_filter)
+        else:
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+
+        if self.dim_ordering == 'th':
+            self.W_shape1 = (self.nb_filter, self.nb_filter,
+                             self.nb_row, self.nb_col)
+        elif self.dim_ordering == 'tf':
+            self.W_shape1 = (self.nb_row, self.nb_col,
+                             self.nb_filter, self.nb_filter)
+
+        if self.stateful:
+            self.reset_states()
+        else:
+            # initial states: 2 all-zero tensor of shape (nb_filter)
+            self.states = [None, None, None, None]
+
+        self.W_i = self.init(self.W_shape, name='{}_W_i'.format(self.name))
+        self.U_i = self.inner_init(self.W_shape1,
+                                   name='{}_U_i'.format(self.name))
+        self.b_i = K.zeros((self.nb_filter,), name='{}_b_i'.format(self.name))
+
+        self.W_f = self.init(self.W_shape, name='{}_W_f'.format(self.name))
+        self.U_f = self.inner_init(self.W_shape1,
+                                   name='{}_U_f'.format(self.name))
+        self.b_f = self.forget_bias_init((self.nb_filter,),
+                                         name='{}_b_f'.format(self.name))
+
+        self.W_c = self.init(self.W_shape, name='{}_W_c'.format(self.name))
+        self.U_c = self.inner_init(self.W_shape1,
+                                   name='{}_U_c'.format(self.name))
+        self.b_c = K.zeros((self.nb_filter,), name='{}_b_c'.format(self.name))
+
+        self.W_o = self.init(self.W_shape, name='{}_W_o'.format(self.name))
+        self.U_o = self.inner_init(self.W_shape1,
+                                   name='{}_U_o'.format(self.name))
+        self.b_o = K.zeros((self.nb_filter,), name='{}_b_o'.format(self.name))
+
+        self.trainable_weights = [self.W_i, self.U_i, self.b_i,
+                                  self.W_c, self.U_c, self.b_c,
+                                  self.W_f, self.U_f, self.b_f,
+                                  self.W_o, self.U_o, self.b_o]
+
+        self.W = K.concatenate([self.W_i, self.W_f, self.W_c, self.W_o])
+        self.U = K.concatenate([self.U_i, self.U_f, self.U_c, self.U_o])
+        self.b = K.concatenate([self.b_i, self.b_f, self.b_c, self.b_o])
+
+        self.regularizers = []
+        if self.W_regularizer:
+            self.W_regularizer.set_param(self.W)
+            self.regularizers.append(self.W_regularizer)
+        if self.U_regularizer:
+            self.U_regularizer.set_param(self.U)
+            self.regularizers.append(self.U_regularizer)
+        if self.b_regularizer:
+            self.b_regularizer.set_param(self.b)
+            self.regularizers.append(self.b_regularizer)
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+        self.built = True
+
+    def reset_states(self):
+        assert self.stateful, 'Layer must be stateful.'
+        input_shape = self.input_spec[0].shape
+        output_shape = self.get_output_shape_for(input_shape)
+        if not input_shape[0]:
+            raise ValueError('If a RNN is stateful, a complete ' +
+                             'input_shape must be provided ' +
+                             '(including batch size).')
+
+        if self.return_sequences:
+            out_row, out_col, out_filter = output_shape[2:]
+        else:
+            out_row, out_col, out_filter = output_shape[1:]
+
+        if hasattr(self, 'states'):
+            K.set_value(self.states[0],
+                        np.zeros((input_shape[0],
+                                  out_row, out_col, out_filter)))
+            K.set_value(self.states[1],
+                        np.zeros((input_shape[0],
+                                  out_row, out_col, out_filter)))
+        else:
+            self.states = [K.zeros((input_shape[0],
+                                    out_row, out_col, out_filter)),
+                           K.zeros((input_shape[0],
+                                    out_row, out_col, out_filter))]
+
+    def conv_step(self, x, W, b=None, border_mode='valid'):
+        input_shape = self.input_spec[0].shape
+
+        conv_out = K.conv2d(x, W, strides=self.subsample,
+                            border_mode=border_mode,
+                            dim_ordering=self.dim_ordering,
+                            image_shape=(input_shape[0],
+                                         input_shape[2],
+                                         input_shape[3],
+                                         input_shape[4]),
+                            filter_shape=self.W_shape)
+        if b:
+            if self.dim_ordering == 'th':
+                conv_out = conv_out + K.reshape(b, (1, self.nb_filter, 1, 1))
+            elif self.dim_ordering == 'tf':
+                conv_out = conv_out + K.reshape(b, (1, 1, 1, self.nb_filter))
+            else:
+                raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+
+        return conv_out
+
+    def conv_step_hidden(self, x, W, border_mode='valid'):
+        # This new function was defined because the
+        # image shape must be hardcoded
+        input_shape = self.input_spec[0].shape
+        output_shape = self.get_output_shape_for(input_shape)
+        if self.return_sequences:
+            out_row, out_col, out_filter = output_shape[2:]
+        else:
+            out_row, out_col, out_filter = output_shape[1:]
+
+        conv_out = K.conv2d(x, W, strides=(1, 1),
+                            border_mode=border_mode,
+                            dim_ordering=self.dim_ordering,
+                            image_shape=(input_shape[0],
+                                         out_row, out_col,
+                                         out_filter),
+                            filter_shape=self.W_shape1)
+
+        return conv_out
+
+    def step(self, x, states):
+        assert len(states) == 4
+        h_tm1 = states[0]
+        c_tm1 = states[1]
+        B_U = states[2]
+        B_W = states[3]
+
+        x_i = self.conv_step(x * B_W[0], self.W_i, self.b_i,
+                             border_mode=self.border_mode)
+        x_f = self.conv_step(x * B_W[1], self.W_f, self.b_f,
+                             border_mode=self.border_mode)
+        x_c = self.conv_step(x * B_W[2], self.W_c, self.b_c,
+                             border_mode=self.border_mode)
+        x_o = self.conv_step(x * B_W[3], self.W_o, self.b_o,
+                             border_mode=self.border_mode)
+
+        # U : from nb_filter to nb_filter
+        # Same because must be stable in the output space
+        h_i = self.conv_step_hidden(h_tm1 * B_U[0], self.U_i,
+                                    border_mode='same')
+        h_f = self.conv_step_hidden(h_tm1 * B_U[1], self.U_f,
+                                    border_mode='same')
+        h_c = self.conv_step_hidden(h_tm1 * B_U[2], self.U_c,
+                                    border_mode='same')
+        h_o = self.conv_step_hidden(h_tm1 * B_U[3], self.U_o,
+                                    border_mode='same')
+
+        i = self.inner_activation(x_i + h_i)
+        f = self.inner_activation(x_f + h_f)
+        c = f * c_tm1 + i * self.activation(x_c + h_c)
+        o = self.inner_activation(x_o + h_o)
+        h = o * self.activation(c)
+
+        return h, [h, c]
+
+    def get_constants(self, x):
+        constants = []
+        if 0 < self.dropout_U < 1:
+            ones = K.zeros_like(x)
+            ones = K.sum(ones, axis=1)
+            ones = self.conv_step(ones, K.zeros(self.W_shape),
+                                  border_mode=self.border_mode)
+            ones += 1
+            B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones)
+                   for _ in range(4)]
+            constants.append(B_U)
+        else:
+            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
+
+        if 0 < self.dropout_W < 1:
+            ones = K.zeros_like(x)
+            ones = K.sum(ones, axis=1)
+            ones += 1
+            B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones)
+                   for _ in range(4)]
+            constants.append(B_W)
+        else:
+            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
+        return constants
+
+    def get_config(self):
+        config = {'nb_filter': self.nb_filter,
+                  'nb_row': self.nb_row,
+                  'nb_col': self.nb_col,
+                  'init': self.init.__name__,
+                  'inner_init': self.inner_init.__name__,
+                  'forget_bias_init': self.forget_bias_init.__name__,
+                  'activation': self.activation.__name__,
+                  'dim_ordering': self.dim_ordering,
+                  'border_mode': self.border_mode,
+                  'inner_activation': self.inner_activation.__name__}
+        base_config = super(ConvLSTM2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

keras/layers/embeddings.py

@@ -1,12 +1,14 @@
 from __future__ import absolute_import
 
 from .. import backend as K
-from .. import initializations, regularizers, constraints
+from .. import initializations
+from .. import regularizers
+from .. import constraints
 from ..engine import Layer
 
 
 class Embedding(Layer):
-    '''Turn positive integers (indexes) into dense vectors of fixed size.
+    """Turn positive integers (indexes) into dense vectors of fixed size.
     eg. [[4], [20]] -> [[0.25, 0.1], [0.6, -0.2]]
 
     This layer can only be used as the first layer in a model.
@@ -62,8 +64,7 @@ class Embedding(Layer):
 
     # References
         - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](http://arxiv.org/abs/1512.05287)
-    '''
-    input_ndim = 2
+    """
 
     def __init__(self, input_dim, output_dim,
                  init='uniform', input_length=None,
@@ -91,25 +92,15 @@ class Embedding(Layer):
         super(Embedding, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        self.W = self.init((self.input_dim, self.output_dim),
-                           name='{}_W'.format(self.name))
-        self.trainable_weights = [self.W]
-
-        self.constraints = {}
-        if self.W_constraint:
-            self.constraints[self.W] = self.W_constraint
-
-        self.regularizers = []
-        if self.W_regularizer:
-            self.W_regularizer.set_param(self.W)
-            self.regularizers.append(self.W_regularizer)
-
-        if self.activity_regularizer:
-            self.activity_regularizer.set_layer(self)
-            self.regularizers.append(self.activity_regularizer)
+        self.W = self.add_weight((self.input_dim, self.output_dim),
+                                 initializer=self.init,
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer,
+                                 constraint=self.W_constraint)
 
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
+        self.built = True
 
     def compute_mask(self, x, mask=None):
         if not self.mask_zero:

keras/layers/local.py

@@ -1,14 +1,20 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
-from keras import backend as K
-from keras.layers import activations, initializations, regularizers, constraints
-from keras.engine import Layer, InputSpec
+from .. import backend as K
+from .. import activations
+from .. import initializations
+from .. import regularizers
+from .. import constraints
+from ..engine import Layer
+from ..engine import InputSpec
 from ..utils.np_utils import conv_output_length
 
 
 class LocallyConnected1D(Layer):
-    '''The `LocallyConnected1D` layer works similarly to
+    """Locally-connected layer for 1D inputs.
+
+    The `LocallyConnected1D` layer works similarly to
     the `Convolution1D` layer, except that weights are unshared,
     that is, a different set of filters is applied at each different patch
     of the input.
@@ -73,16 +79,17 @@ class LocallyConnected1D(Layer):
     # Output shape
         3D tensor with shape: `(samples, new_steps, nb_filter)`.
         `steps` value might have changed due to padding.
-    '''
+    """
+
     def __init__(self, nb_filter, filter_length,
-                 init='uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample_length=1,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, input_dim=None, input_length=None, **kwargs):
         if border_mode != 'valid':
-            raise Exception('Invalid border mode for LocallyConnected1D '
-                            '(only "valid" is supported):', border_mode)
+            raise ValueError('Invalid border mode for LocallyConnected1D '
+                             '(only "valid" is supported):', border_mode)
         self.nb_filter = nb_filter
         self.filter_length = filter_length
         self.init = initializations.get(init, dim_ordering='th')
@@ -110,35 +117,27 @@ class LocallyConnected1D(Layer):
     def build(self, input_shape):
         input_dim = input_shape[2]
         _, output_length, nb_filter = self.get_output_shape_for(input_shape)
-
-        self.W_shape = (output_length, self.filter_length * input_dim, nb_filter)
-        self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
+        self.W_shape = (output_length,
+                        self.filter_length * input_dim,
+                        nb_filter)
+        self.W = self.add_weight(self.W_shape,
+                                 initializer=self.init,
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer,
+                                 constraint=self.W_constraint)
         if self.bias:
-            self.b = K.zeros((output_length, self.nb_filter), name='{}_b'.format(self.name))
-            self.trainable_weights = [self.W, self.b]
+            self.b = self.add_weight((output_length, self.nb_filter),
+                                     initializer='zero',
+                                     name='{}_b'.format(self.name),
+                                     regularizer=self.b_regularizer,
+                                     constraint=self.b_constraint)
         else:
-            self.trainable_weights = [self.W]
-
-        self.regularizers = []
-        if self.W_regularizer:
-            self.W_regularizer.set_param(self.W)
-            self.regularizers.append(self.W_regularizer)
-        if self.b_regularizer:
-            self.b_regularizer.set_param(self.b)
-            self.regularizers.append(self.b_regularizer)
-        if self.activity_regularizer:
-            self.activity_regularizer.set_layer(self)
-            self.regularizers.append(self.activity_regularizer)
-
-        self.constraints = {}
-        if self.W_constraint:
-            self.constraints[self.W] = self.W_constraint
-        if self.b_constraint:
-            self.constraints[self.b] = self.b_constraint
+            self.b = None
 
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         length = conv_output_length(input_shape[1],
@@ -186,7 +185,9 @@ class LocallyConnected1D(Layer):
 
 
 class LocallyConnected2D(Layer):
-    '''The `LocallyConnected2D` layer works similarly
+    """Locally-connected layer for 2D inputs.
+
+    The `LocallyConnected2D` layer works similarly
     to the `Convolution2D` layer, except that weights are unshared,
     that is, a different set of filters is applied at each
     different patch of the input.
@@ -255,9 +256,10 @@ class LocallyConnected2D(Layer):
         or 4D tensor with shape:
         `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'.
         `rows` and `cols` values might have changed due to padding.
-    '''
+    """
+
     def __init__(self, nb_filter, nb_row, nb_col,
-                 init='glorot_uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample=(1, 1),
                  dim_ordering='default',
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
@@ -266,8 +268,8 @@ class LocallyConnected2D(Layer):
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
         if border_mode != 'valid':
-            raise Exception('Invalid border mode for LocallyConnected2D '
-                            '(only "valid" is supported):', border_mode)
+            raise ValueError('Invalid border mode for LocallyConnected2D '
+                             '(only "valid" is supported):', border_mode)
         self.nb_filter = nb_filter
         self.nb_row = nb_row
         self.nb_col = nb_col
@@ -276,7 +278,8 @@ class LocallyConnected2D(Layer):
 
         self.border_mode = border_mode
         self.subsample = tuple(subsample)
-        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+        if dim_ordering not in {'tf', 'th'}:
+            raise ValueError('`dim_ordering` must be in {tf, th}.')
         self.dim_ordering = dim_ordering
 
         self.W_regularizer = regularizers.get(W_regularizer)
@@ -300,39 +303,31 @@ class LocallyConnected2D(Layer):
             _, output_row, output_col, nb_filter = output_shape
             input_filter = input_shape[3]
         else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
 
         self.output_row = output_row
         self.output_col = output_col
-        self.W_shape = (output_row * output_col, self.nb_row * self.nb_col * input_filter, nb_filter)
-        self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
-
+        self.W_shape = (output_row * output_col,
+                        self.nb_row * self.nb_col * input_filter,
+                        nb_filter)
+        self.W = self.add_weight(self.W_shape,
+                                 initializer=self.init,
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer,
+                                 constraint=self.W_constraint)
         if self.bias:
-            self.b = K.zeros((output_row, output_col, nb_filter), name='{}_b'.format(self.name))
-            self.trainable_weights = [self.W, self.b]
+            self.b = self.add_weight((output_row, output_col, nb_filter),
+                                     initializer='zero',
+                                     name='{}_b'.format(self.name),
+                                     regularizer=self.b_regularizer,
+                                     constraint=self.b_constraint)
         else:
-            self.trainable_weights = [self.W]
-
-        self.regularizers = []
-        if self.W_regularizer:
-            self.W_regularizer.set_param(self.W)
-            self.regularizers.append(self.W_regularizer)
-        if self.bias and self.b_regularizer:
-            self.b_regularizer.set_param(self.b)
-            self.regularizers.append(self.b_regularizer)
-        if self.activity_regularizer:
-            self.activity_regularizer.set_layer(self)
-            self.regularizers.append(self.activity_regularizer)
-
-        self.constraints = {}
-        if self.W_constraint:
-            self.constraints[self.W] = self.W_constraint
-        if self.bias and self.b_constraint:
-            self.constraints[self.b] = self.b_constraint
+            self.b = None
 
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         if self.dim_ordering == 'th':
@@ -342,7 +337,7 @@ class LocallyConnected2D(Layer):
             rows = input_shape[1]
             cols = input_shape[2]
         else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
 
         rows = conv_output_length(rows, self.nb_row,
                                   self.border_mode, self.subsample[0])
@@ -353,15 +348,13 @@ class LocallyConnected2D(Layer):
             return (input_shape[0], self.nb_filter, rows, cols)
         elif self.dim_ordering == 'tf':
             return (input_shape[0], rows, cols, self.nb_filter)
-        else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
 
     def call(self, x, mask=None):
         stride_row, stride_col = self.subsample
         _, feature_dim, nb_filter = self.W_shape
 
         if self.dim_ordering == 'th':
-            if K._backend == 'theano':
+            if K.backend() == 'theano':
                 output = []
                 for i in range(self.output_row):
                     for j in range(self.output_col):
@@ -399,15 +392,13 @@ class LocallyConnected2D(Layer):
             output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter))
             output = K.permute_dimensions(output, (2, 0, 1, 3))
         else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
 
         if self.bias:
             if self.dim_ordering == 'th':
                 output += K.reshape(self.b, (1, nb_filter, self.output_row, self.output_col))
             elif self.dim_ordering == 'tf':
                 output += K.reshape(self.b, (1, self.output_row, self.output_col, nb_filter))
-            else:
-                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
 
         output = self.activation(output)
         return output

keras/layers/noise.py

@@ -5,9 +5,10 @@ import numpy as np
 
 
 class GaussianNoise(Layer):
-    '''Apply to the input an additive zero-centered Gaussian noise with
-    standard deviation `sigma`. This is useful to mitigate overfitting
-    (you could see it as a kind of random data augmentation).
+    """Apply additive zero-centered Gaussian noise.
+
+    This is useful to mitigate overfitting
+    (you could see it as a form of random data augmentation).
     Gaussian Noise (GS) is a natural choice as corruption process
     for real valued inputs.
 
@@ -23,7 +24,8 @@ class GaussianNoise(Layer):
 
     # Output shape
         Same shape as input.
-    '''
+    """
+
     def __init__(self, sigma, **kwargs):
         self.supports_masking = True
         self.sigma = sigma
@@ -43,13 +45,14 @@ class GaussianNoise(Layer):
 
 
 class GaussianDropout(Layer):
-    '''Apply to the input an multiplicative one-centered Gaussian noise
-    with standard deviation `sqrt(p/(1-p))`.
+    """Apply multiplicative 1-centered Gaussian noise.
 
     As it is a regularization layer, it is only active at training time.
 
     # Arguments
         p: float, drop probability (as with `Dropout`).
+            The multiplicative noise will have
+            standard deviation `sqrt(p / (1 - p))`.
 
     # Input shape
         Arbitrary. Use the keyword argument `input_shape`
@@ -60,8 +63,9 @@ class GaussianDropout(Layer):
         Same shape as input.
 
     # References
-        [Dropout: A Simple Way to Prevent Neural Networks from Overfitting Srivastava, Hinton, et al. 2014](http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf)
-    '''
+        - [Dropout: A Simple Way to Prevent Neural Networks from Overfitting Srivastava, Hinton, et al. 2014](http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf)
+    """
+
     def __init__(self, p, **kwargs):
         self.supports_masking = True
         self.p = p

keras/layers/normalization.py

@@ -4,12 +4,15 @@ from .. import backend as K
 
 
 class BatchNormalization(Layer):
-    '''Normalize the activations of the previous layer at each batch,
+    """Batch normalization layer (Ioffe and Szegedy, 2014).
+
+    Normalize the activations of the previous layer at each batch,
     i.e. applies a transformation that maintains the mean activation
     close to 0 and the activation standard deviation close to 1.
 
     # Arguments
         epsilon: small float > 0. Fuzz parameter.
+            Theano expects epsilon >= 1e-5.
         mode: integer, 0, 1 or 2.
             - 0: feature-wise normalization.
                 Each feature map in the input will
@@ -58,9 +61,10 @@ class BatchNormalization(Layer):
         Same shape as input.
 
     # References
-        - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](http://jmlr.org/proceedings/papers/v37/ioffe15.html)
-    '''
-    def __init__(self, epsilon=1e-5, mode=0, axis=-1, momentum=0.99,
+        - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](https://arxiv.org/abs/1502.03167)
+    """
+
+    def __init__(self, epsilon=1e-3, mode=0, axis=-1, momentum=0.99,
                  weights=None, beta_init='zero', gamma_init='one',
                  gamma_regularizer=None, beta_regularizer=None, **kwargs):
         self.supports_masking = True
@@ -81,66 +85,45 @@ class BatchNormalization(Layer):
         self.input_spec = [InputSpec(shape=input_shape)]
         shape = (input_shape[self.axis],)
 
-        self.gamma = self.gamma_init(shape, name='{}_gamma'.format(self.name))
-        self.beta = self.beta_init(shape, name='{}_beta'.format(self.name))
-        self.trainable_weights = [self.gamma, self.beta]
-
-        self.regularizers = []
-        if self.gamma_regularizer:
-            self.gamma_regularizer.set_param(self.gamma)
-            self.regularizers.append(self.gamma_regularizer)
-
-        if self.beta_regularizer:
-            self.beta_regularizer.set_param(self.beta)
-            self.regularizers.append(self.beta_regularizer)
-
-        self.running_mean = K.zeros(shape,
-                                    name='{}_running_mean'.format(self.name))
-        self.running_std = K.ones(shape,
-                                  name='{}_running_std'.format(self.name))
-        self.non_trainable_weights = [self.running_mean, self.running_std]
+        self.gamma = self.add_weight(shape,
+                                     initializer=self.gamma_init,
+                                     regularizer=self.gamma_regularizer,
+                                     name='{}_gamma'.format(self.name))
+        self.beta = self.add_weight(shape,
+                                    initializer=self.beta_init,
+                                    regularizer=self.beta_regularizer,
+                                    name='{}_beta'.format(self.name))
+        self.running_mean = self.add_weight(shape, initializer='zero',
+                                            name='{}_running_mean'.format(self.name),
+                                            trainable=False)
+        self.running_std = self.add_weight(shape, initializer='one',
+                                           name='{}_running_std'.format(self.name),
+                                           trainable=False)
 
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
         self.built = True
-        self.called_with = None
 
     def call(self, x, mask=None):
         if self.mode == 0 or self.mode == 2:
             assert self.built, 'Layer must be built before being called'
-            input_shape = self.input_spec[0].shape
+            input_shape = K.int_shape(x)
 
             reduction_axes = list(range(len(input_shape)))
             del reduction_axes[self.axis]
             broadcast_shape = [1] * len(input_shape)
             broadcast_shape[self.axis] = input_shape[self.axis]
 
-            if self.mode == 2:
-                x_normed, mean, std = K.normalize_batch_in_training(
-                    x, self.gamma, self.beta, reduction_axes,
-                    epsilon=self.epsilon)
-            else:
-                # mode 0
-                if self.called_with not in {None, x}:
-                    raise Exception('You are attempting to share a '
-                                    'same `BatchNormalization` layer across '
-                                    'different data flows. '
-                                    'This is not possible. '
-                                    'You should use `mode=2` in '
-                                    '`BatchNormalization`, which has '
-                                    'a similar behavior but is shareable '
-                                    '(see docs for a description of '
-                                    'the behavior).')
-                self.called_with = x
-                x_normed, mean, std = K.normalize_batch_in_training(
-                    x, self.gamma, self.beta, reduction_axes,
-                    epsilon=self.epsilon)
+            x_normed, mean, std = K.normalize_batch_in_training(
+                x, self.gamma, self.beta, reduction_axes,
+                epsilon=self.epsilon)
 
-                self.updates = [K.moving_average_update(self.running_mean, mean, self.momentum),
-                                K.moving_average_update(self.running_std, std, self.momentum)]
+            if self.mode == 0:
+                self.add_update([K.moving_average_update(self.running_mean, mean, self.momentum),
+                                 K.moving_average_update(self.running_std, std, self.momentum)], x)
 
-                if K.backend() == 'tensorflow' and sorted(reduction_axes) == range(K.ndim(x))[:-1]:
+                if sorted(reduction_axes) == range(K.ndim(x))[:-1]:
                     x_normed_running = K.batch_normalization(
                         x, self.running_mean, self.running_std,
                         self.beta, self.gamma,
@@ -168,11 +151,11 @@ class BatchNormalization(Layer):
         return x_normed
 
     def get_config(self):
-        config = {"epsilon": self.epsilon,
-                  "mode": self.mode,
-                  "axis": self.axis,
-                  "gamma_regularizer": self.gamma_regularizer.get_config() if self.gamma_regularizer else None,
-                  "beta_regularizer": self.beta_regularizer.get_config() if self.beta_regularizer else None,
-                  "momentum": self.momentum}
+        config = {'epsilon': self.epsilon,
+                  'mode': self.mode,
+                  'axis': self.axis,
+                  'gamma_regularizer': self.gamma_regularizer.get_config() if self.gamma_regularizer else None,
+                  'beta_regularizer': self.beta_regularizer.get_config() if self.beta_regularizer else None,
+                  'momentum': self.momentum}
         base_config = super(BatchNormalization, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))

keras/layers/pooling.py

@@ -2,14 +2,14 @@
 from __future__ import absolute_import
 
 from .. import backend as K
-from ..engine import Layer, InputSpec
+from ..engine import Layer
+from ..engine import InputSpec
 from ..utils.np_utils import conv_output_length
 
 
 class _Pooling1D(Layer):
-    '''Abstract class for different pooling 1D layers.
-    '''
-    input_dim = 3
+    """Abstract class for different pooling 1D layers.
+    """
 
     def __init__(self, pool_length=2, stride=None,
                  border_mode='valid', **kwargs):
@@ -20,7 +20,8 @@ class _Pooling1D(Layer):
         self.stride = stride
         self.st = (self.stride, 1)
         self.pool_size = (pool_length, 1)
-        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
+        if border_mode not in {'valid', 'same'}:
+            raise ValueError('`border_mode` must be in {valid, same}.')
         self.border_mode = border_mode
         self.input_spec = [InputSpec(ndim=3)]
 
@@ -29,19 +30,17 @@ class _Pooling1D(Layer):
                                     self.border_mode, self.stride)
         return (input_shape[0], length, input_shape[2])
 
-    def _pooling_function(self, back_end, inputs, pool_size, strides,
+    def _pooling_function(self, inputs, pool_size, strides,
                           border_mode, dim_ordering):
         raise NotImplementedError
 
     def call(self, x, mask=None):
-        x = K.expand_dims(x, -1)   # add dummy last dimension
-        x = K.permute_dimensions(x, (0, 2, 1, 3))
+        x = K.expand_dims(x, 2)   # add dummy last dimension
         output = self._pooling_function(inputs=x, pool_size=self.pool_size,
                                         strides=self.st,
                                         border_mode=self.border_mode,
-                                        dim_ordering='th')
-        output = K.permute_dimensions(output, (0, 2, 1, 3))
-        return K.squeeze(output, 3)  # remove dummy last dimension
+                                        dim_ordering='tf')
+        return K.squeeze(output, 2)  # remove dummy last dimension
 
     def get_config(self):
         config = {'stride': self.stride,
@@ -52,7 +51,7 @@ class _Pooling1D(Layer):
 
 
 class MaxPooling1D(_Pooling1D):
-    '''Max pooling operation for temporal data.
+    """Max pooling operation for temporal data.
 
     # Input shape
         3D tensor with shape: `(samples, steps, features)`.
@@ -66,8 +65,7 @@ class MaxPooling1D(_Pooling1D):
             2 will halve the input.
             If None, it will default to `pool_length`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
-    '''
+    """
 
     def __init__(self, pool_length=2, stride=None,
                  border_mode='valid', **kwargs):
@@ -82,21 +80,20 @@ class MaxPooling1D(_Pooling1D):
 
 
 class AveragePooling1D(_Pooling1D):
-    '''Average pooling for temporal data.
+    """Average pooling for temporal data.
 
     # Arguments
         pool_length: factor by which to downscale. 2 will halve the input.
         stride: integer, or None. Stride value.
             If None, it will default to `pool_length`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
 
     # Input shape
         3D tensor with shape: `(samples, steps, features)`.
 
     # Output shape
         3D tensor with shape: `(samples, downsampled_steps, features)`.
-    '''
+    """
 
     def __init__(self, pool_length=2, stride=None,
                  border_mode='valid', **kwargs):
@@ -111,8 +108,8 @@ class AveragePooling1D(_Pooling1D):
 
 
 class _Pooling2D(Layer):
-    '''Abstract class for different pooling 2D layers.
-    '''
+    """Abstract class for different pooling 2D layers.
+    """
 
     def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
                  dim_ordering='default', **kwargs):
@@ -123,9 +120,11 @@ class _Pooling2D(Layer):
         if strides is None:
             strides = self.pool_size
         self.strides = tuple(strides)
-        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
+        if border_mode not in {'valid', 'same'}:
+            raise ValueError('`border_mode` must be in {valid, same}.')
         self.border_mode = border_mode
-        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+        if dim_ordering not in {'tf', 'th'}:
+            raise ValueError('`dim_ordering` must be in {tf, th}.')
         self.dim_ordering = dim_ordering
         self.input_spec = [InputSpec(ndim=4)]
 
@@ -137,7 +136,7 @@ class _Pooling2D(Layer):
             rows = input_shape[1]
             cols = input_shape[2]
         else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
 
         rows = conv_output_length(rows, self.pool_size[0],
                                   self.border_mode, self.strides[0])
@@ -148,15 +147,14 @@ class _Pooling2D(Layer):
             return (input_shape[0], input_shape[1], rows, cols)
         elif self.dim_ordering == 'tf':
             return (input_shape[0], rows, cols, input_shape[3])
-        else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
 
     def _pooling_function(self, inputs, pool_size, strides,
                           border_mode, dim_ordering):
         raise NotImplementedError
 
     def call(self, x, mask=None):
-        output = self._pooling_function(inputs=x, pool_size=self.pool_size,
+        output = self._pooling_function(inputs=x,
+                                        pool_size=self.pool_size,
                                         strides=self.strides,
                                         border_mode=self.border_mode,
                                         dim_ordering=self.dim_ordering)
@@ -172,7 +170,7 @@ class _Pooling2D(Layer):
 
 
 class MaxPooling2D(_Pooling2D):
-    '''Max pooling operation for spatial data.
+    """Max pooling operation for spatial data.
 
     # Arguments
         pool_size: tuple of 2 integers,
@@ -181,12 +179,11 @@ class MaxPooling2D(_Pooling2D):
         strides: tuple of 2 integers, or None. Strides values.
             If None, it will default to `pool_size`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -199,7 +196,7 @@ class MaxPooling2D(_Pooling2D):
         `(nb_samples, channels, pooled_rows, pooled_cols)` if dim_ordering='th'
         or 4D tensor with shape:
         `(samples, pooled_rows, pooled_cols, channels)` if dim_ordering='tf'.
-    '''
+    """
 
     def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
                  dim_ordering='default', **kwargs):
@@ -209,12 +206,13 @@ class MaxPooling2D(_Pooling2D):
     def _pooling_function(self, inputs, pool_size, strides,
                           border_mode, dim_ordering):
         output = K.pool2d(inputs, pool_size, strides,
-                          border_mode, dim_ordering, pool_mode='max')
+                          border_mode, dim_ordering,
+                          pool_mode='max')
         return output
 
 
 class AveragePooling2D(_Pooling2D):
-    '''Average pooling operation for spatial data.
+    """Average pooling operation for spatial data.
 
     # Arguments
         pool_size: tuple of 2 integers,
@@ -223,12 +221,11 @@ class AveragePooling2D(_Pooling2D):
         strides: tuple of 2 integers, or None. Strides values.
             If None, it will default to `pool_size`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -241,7 +238,7 @@ class AveragePooling2D(_Pooling2D):
         `(nb_samples, channels, pooled_rows, pooled_cols)` if dim_ordering='th'
         or 4D tensor with shape:
         `(samples, pooled_rows, pooled_cols, channels)` if dim_ordering='tf'.
-    '''
+    """
 
     def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
                  dim_ordering='default', **kwargs):
@@ -256,8 +253,8 @@ class AveragePooling2D(_Pooling2D):
 
 
 class _Pooling3D(Layer):
-    '''Abstract class for different pooling 3D layers.
-    '''
+    """Abstract class for different pooling 3D layers.
+    """
 
     def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
                  dim_ordering='default', **kwargs):
@@ -268,9 +265,11 @@ class _Pooling3D(Layer):
         if strides is None:
             strides = self.pool_size
         self.strides = tuple(strides)
-        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
+        if border_mode not in {'valid', 'same'}:
+            raise ValueError('`border_mode` must be in {valid, same}.')
         self.border_mode = border_mode
-        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+        if dim_ordering not in {'tf', 'th'}:
+            raise ValueError('`dim_ordering` must be in {tf, th}.')
         self.dim_ordering = dim_ordering
         self.input_spec = [InputSpec(ndim=5)]
 
@@ -284,7 +283,7 @@ class _Pooling3D(Layer):
             len_dim2 = input_shape[2]
             len_dim3 = input_shape[3]
         else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
 
         len_dim1 = conv_output_length(len_dim1, self.pool_size[0],
                                       self.border_mode, self.strides[0])
@@ -292,13 +291,14 @@ class _Pooling3D(Layer):
                                       self.border_mode, self.strides[1])
         len_dim3 = conv_output_length(len_dim3, self.pool_size[2],
                                       self.border_mode, self.strides[2])
-
         if self.dim_ordering == 'th':
-            return (input_shape[0], input_shape[1], len_dim1, len_dim2, len_dim3)
+            return (input_shape[0],
+                    input_shape[1],
+                    len_dim1, len_dim2, len_dim3)
         elif self.dim_ordering == 'tf':
-            return (input_shape[0], len_dim1, len_dim2, len_dim3, input_shape[4])
-        else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+            return (input_shape[0],
+                    len_dim1, len_dim2, len_dim3,
+                    input_shape[4])
 
     def _pooling_function(self, inputs, pool_size, strides,
                           border_mode, dim_ordering):
@@ -321,7 +321,7 @@ class _Pooling3D(Layer):
 
 
 class MaxPooling3D(_Pooling3D):
-    '''Max pooling operation for 3D data (spatial or spatio-temporal).
+    """Max pooling operation for 3D data (spatial or spatio-temporal).
 
     # Arguments
         pool_size: tuple of 3 integers,
@@ -333,7 +333,7 @@ class MaxPooling3D(_Pooling3D):
             (the depth) is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         5D tensor with shape:
@@ -346,7 +346,7 @@ class MaxPooling3D(_Pooling3D):
         `(nb_samples, channels, pooled_dim1, pooled_dim2, pooled_dim3)` if dim_ordering='th'
         or 5D tensor with shape:
         `(samples, pooled_dim1, pooled_dim2, pooled_dim3, channels)` if dim_ordering='tf'.
-    '''
+    """
 
     def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
                  dim_ordering='default', **kwargs):
@@ -361,7 +361,7 @@ class MaxPooling3D(_Pooling3D):
 
 
 class AveragePooling3D(_Pooling3D):
-    '''Average pooling operation for 3D data (spatial or spatio-temporal).
+    """Average pooling operation for 3D data (spatial or spatio-temporal).
 
     # Arguments
         pool_size: tuple of 3 integers,
@@ -373,7 +373,7 @@ class AveragePooling3D(_Pooling3D):
             (the depth) is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         5D tensor with shape:
@@ -386,7 +386,7 @@ class AveragePooling3D(_Pooling3D):
         `(nb_samples, channels, pooled_dim1, pooled_dim2, pooled_dim3)` if dim_ordering='th'
         or 5D tensor with shape:
         `(samples, pooled_dim1, pooled_dim2, pooled_dim3, channels)` if dim_ordering='tf'.
-    '''
+    """
 
     def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
                  dim_ordering='default', **kwargs):
@@ -396,11 +396,14 @@ class AveragePooling3D(_Pooling3D):
     def _pooling_function(self, inputs, pool_size, strides,
                           border_mode, dim_ordering):
         output = K.pool3d(inputs, pool_size, strides,
-                          border_mode, dim_ordering, pool_mode='avg')
+                          border_mode, dim_ordering,
+                          pool_mode='avg')
         return output
 
 
 class _GlobalPooling1D(Layer):
+    """Abstract class for different global pooling 1D layers.
+    """
 
     def __init__(self, **kwargs):
         super(_GlobalPooling1D, self).__init__(**kwargs)
@@ -414,40 +417,41 @@ class _GlobalPooling1D(Layer):
 
 
 class GlobalAveragePooling1D(_GlobalPooling1D):
-    '''Global average pooling operation for temporal data.
+    """Global average pooling operation for temporal data.
 
     # Input shape
         3D tensor with shape: `(samples, steps, features)`.
 
     # Output shape
         2D tensor with shape: `(samples, features)`.
-    '''
+    """
 
     def call(self, x, mask=None):
         return K.mean(x, axis=1)
 
 
 class GlobalMaxPooling1D(_GlobalPooling1D):
-    '''Global max pooling operation for temporal data.
+    """Global max pooling operation for temporal data.
 
     # Input shape
         3D tensor with shape: `(samples, steps, features)`.
 
     # Output shape
         2D tensor with shape: `(samples, features)`.
-    '''
+    """
 
     def call(self, x, mask=None):
         return K.max(x, axis=1)
 
 
 class _GlobalPooling2D(Layer):
+    """Abstract class for different global pooling 2D layers.
+    """
 
     def __init__(self, dim_ordering='default', **kwargs):
         super(_GlobalPooling2D, self).__init__(**kwargs)
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
-        print(dim_ordering)
         self.dim_ordering = dim_ordering
         self.input_spec = [InputSpec(ndim=4)]
 
@@ -467,14 +471,14 @@ class _GlobalPooling2D(Layer):
 
 
 class GlobalAveragePooling2D(_GlobalPooling2D):
-    '''Global average pooling operation for spatial data.
+    """Global average pooling operation for spatial data.
 
     # Arguments
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -485,7 +489,7 @@ class GlobalAveragePooling2D(_GlobalPooling2D):
     # Output shape
         2D tensor with shape:
         `(nb_samples, channels)`
-    '''
+    """
 
     def call(self, x, mask=None):
         if self.dim_ordering == 'tf':
@@ -495,14 +499,14 @@ class GlobalAveragePooling2D(_GlobalPooling2D):
 
 
 class GlobalMaxPooling2D(_GlobalPooling2D):
-    '''Global max pooling operation for spatial data.
+    """Global max pooling operation for spatial data.
 
     # Arguments
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -513,10 +517,92 @@ class GlobalMaxPooling2D(_GlobalPooling2D):
     # Output shape
         2D tensor with shape:
         `(nb_samples, channels)`
-    '''
+    """
 
     def call(self, x, mask=None):
         if self.dim_ordering == 'tf':
             return K.max(x, axis=[1, 2])
         else:
             return K.max(x, axis=[2, 3])
+
+
+class _GlobalPooling3D(Layer):
+    """Abstract class for different global pooling 3D layers.
+    """
+
+    def __init__(self, dim_ordering='default', **kwargs):
+        super(_GlobalPooling3D, self).__init__(**kwargs)
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        self.dim_ordering = dim_ordering
+        self.input_spec = [InputSpec(ndim=5)]
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'tf':
+            return (input_shape[0], input_shape[4])
+        else:
+            return (input_shape[0], input_shape[1])
+
+    def call(self, x, mask=None):
+        raise NotImplementedError
+
+    def get_config(self):
+        config = {'dim_ordering': self.dim_ordering}
+        base_config = super(_GlobalPooling3D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class GlobalAveragePooling3D(_GlobalPooling3D):
+    """Global Average pooling operation for 3D data.
+
+    # Arguments
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+
+    # Input shape
+        5D tensor with shape:
+        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        2D tensor with shape:
+        `(nb_samples, channels)`
+    """
+
+    def call(self, x, mask=None):
+        if self.dim_ordering == 'tf':
+            return K.mean(x, axis=[1, 2, 3])
+        else:
+            return K.mean(x, axis=[2, 3, 4])
+
+
+class GlobalMaxPooling3D(_GlobalPooling3D):
+    """Global Max pooling operation for 3D data.
+
+    # Arguments
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+
+    # Input shape
+        5D tensor with shape:
+        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        2D tensor with shape:
+        `(nb_samples, channels)`
+    """
+
+    def call(self, x, mask=None):
+        if self.dim_ordering == 'tf':
+            return K.max(x, axis=[1, 2, 3])
+        else:
+            return K.max(x, axis=[2, 3, 4])

keras/layers/recurrent.py

@@ -3,14 +3,30 @@ from __future__ import absolute_import
 import numpy as np
 
 from .. import backend as K
-from .. import activations, initializations, regularizers
-from ..engine import Layer, InputSpec
+from .. import activations
+from .. import initializations
+from .. import regularizers
+from ..engine import Layer
+from ..engine import InputSpec
 
 
 def time_distributed_dense(x, w, b=None, dropout=None,
                            input_dim=None, output_dim=None, timesteps=None):
-    '''Apply y.w + b for every temporal slice y of x.
-    '''
+    """Apply `y . w + b` for every temporal slice y of x.
+
+    # Arguments
+        x: input tensor.
+        w: weight matrix.
+        b: optional bias vector.
+        dropout: wether to apply dropout (same dropout mask
+            for every temporal slice of the input).
+        input_dim: integer; optional dimensionality of the input.
+        output_dim: integer; optional dimensionality of the output.
+        timesteps: integer; optional number of timesteps.
+
+    # Returns
+        Output tensor.
+    """
     if not input_dim:
         input_dim = K.shape(x)[2]
     if not timesteps:
@@ -29,10 +45,10 @@ def time_distributed_dense(x, w, b=None, dropout=None,
     x = K.reshape(x, (-1, input_dim))
     x = K.dot(x, w)
     if b:
-        x = x + b
+        x += b
     # reshape to 3D tensor
     if K.backend() == 'tensorflow':
-        x = K.reshape(x, K.pack([-1, timesteps, output_dim]))
+        x = K.reshape(x, K.stack([-1, timesteps, output_dim]))
         x.set_shape([None, None, output_dim])
     else:
         x = K.reshape(x, (-1, timesteps, output_dim))
@@ -40,7 +56,7 @@ def time_distributed_dense(x, w, b=None, dropout=None,
 
 
 class Recurrent(Layer):
-    '''Abstract base class for recurrent layers.
+    """Abstract base class for recurrent layers.
     Do not use in a model -- it's not a valid layer!
     Use its children classes `LSTM`, `GRU` and `SimpleRNN` instead.
 
@@ -128,23 +144,24 @@ class Recurrent(Layer):
     # Note on using statefulness in RNNs
         You can set RNN layers to be 'stateful', which means that the states
         computed for the samples in one batch will be reused as initial states
-        for the samples in the next batch.
-        This assumes a one-to-one mapping between
-        samples in different successive batches.
+        for the samples in the next batch. This assumes a one-to-one mapping
+        between samples in different successive batches.
 
         To enable statefulness:
             - specify `stateful=True` in the layer constructor.
             - specify a fixed batch size for your model, by passing
                 if sequential model:
-                  a `batch_input_shape=(...)` to the first layer in your model.
+                  `batch_input_shape=(...)` to the first layer in your model.
                 else for functional model with 1 or more Input layers:
-                  a `batch_shape=(...)` to all the first layers in your model.
+                  `batch_shape=(...)` to all the first layers in your model.
                 This is the expected shape of your inputs *including the batch size*.
                 It should be a tuple of integers, e.g. `(32, 10, 100)`.
+            - specify `shuffle=False` when calling fit().
 
         To reset the states of your model, call `.reset_states()` on either
         a specific layer, or on your entire model.
-    '''
+    """
+
     def __init__(self, weights=None,
                  return_sequences=False, go_backwards=False, stateful=False,
                  unroll=False, consume_less='cpu',
@@ -198,7 +215,19 @@ class Recurrent(Layer):
         # input shape: (nb_samples, time (padded with zeros), input_dim)
         # note that the .build() method of subclasses MUST define
         # self.input_spec with a complete input shape.
-        input_shape = self.input_spec[0].shape
+        input_shape = K.int_shape(x)
+        if self.unroll and input_shape[1] is None:
+            raise ValueError('Cannot unroll a RNN if the '
+                             'time dimension is undefined. \n'
+                             '- If using a Sequential model, '
+                             'specify the time dimension by passing '
+                             'an `input_shape` or `batch_input_shape` '
+                             'argument to your first layer. If your '
+                             'first layer is an Embedding, you can '
+                             'also use the `input_length` argument.\n'
+                             '- If using the functional API, specify '
+                             'the time dimension by passing a `shape` '
+                             'or `batch_shape` argument to your Input layer.')
         if self.stateful:
             initial_states = self.states
         else:
@@ -214,9 +243,10 @@ class Recurrent(Layer):
                                              unroll=self.unroll,
                                              input_length=input_shape[1])
         if self.stateful:
-            self.updates = []
+            updates = []
             for i in range(len(states)):
-                self.updates.append((self.states[i], states[i]))
+                updates.append((self.states[i], states[i]))
+            self.add_update(updates, x)
 
         if self.return_sequences:
             return outputs
@@ -229,7 +259,7 @@ class Recurrent(Layer):
                   'stateful': self.stateful,
                   'unroll': self.unroll,
                   'consume_less': self.consume_less}
-        if self.stateful:
+        if self.stateful and self.input_spec[0].shape:
             config['batch_input_shape'] = self.input_spec[0].shape
         else:
             config['input_dim'] = self.input_dim
@@ -240,7 +270,7 @@ class Recurrent(Layer):
 
 
 class SimpleRNN(Recurrent):
-    '''Fully-connected RNN where the output is to be fed back to input.
+    """Fully-connected RNN where the output is to be fed back to input.
 
     # Arguments
         output_dim: dimension of the internal projections and the final output.
@@ -262,7 +292,8 @@ class SimpleRNN(Recurrent):
 
     # References
         - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](http://arxiv.org/abs/1512.05287)
-    '''
+    """
+
     def __init__(self, output_dim,
                  init='glorot_uniform', inner_init='orthogonal',
                  activation='tanh',
@@ -275,7 +306,8 @@ class SimpleRNN(Recurrent):
         self.W_regularizer = regularizers.get(W_regularizer)
         self.U_regularizer = regularizers.get(U_regularizer)
         self.b_regularizer = regularizers.get(b_regularizer)
-        self.dropout_W, self.dropout_U = dropout_W, dropout_U
+        self.dropout_W = dropout_W
+        self.dropout_U = dropout_U
 
         if self.dropout_W or self.dropout_U:
             self.uses_learning_phase = True
@@ -291,35 +323,38 @@ class SimpleRNN(Recurrent):
         input_dim = input_shape[2]
         self.input_dim = input_dim
 
-        self.W = self.init((input_dim, self.output_dim),
-                           name='{}_W'.format(self.name))
-        self.U = self.inner_init((self.output_dim, self.output_dim),
-                                 name='{}_U'.format(self.name))
-        self.b = K.zeros((self.output_dim,), name='{}_b'.format(self.name))
-
-        self.regularizers = []
-        if self.W_regularizer:
-            self.W_regularizer.set_param(self.W)
-            self.regularizers.append(self.W_regularizer)
-        if self.U_regularizer:
-            self.U_regularizer.set_param(self.U)
-            self.regularizers.append(self.U_regularizer)
-        if self.b_regularizer:
-            self.b_regularizer.set_param(self.b)
-            self.regularizers.append(self.b_regularizer)
-
-        self.trainable_weights = [self.W, self.U, self.b]
+        self.W = self.add_weight((input_dim, self.output_dim),
+                                 initializer=self.init,
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer)
+        self.U = self.add_weight((self.output_dim, self.output_dim),
+                                 initializer=self.inner_init,
+                                 name='{}_U'.format(self.name),
+                                 regularizer=self.U_regularizer)
+        self.b = self.add_weight((self.output_dim,),
+                                 initializer='zero',
+                                 name='{}_b'.format(self.name),
+                                 regularizer=self.b_regularizer)
 
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def reset_states(self):
         assert self.stateful, 'Layer must be stateful.'
         input_shape = self.input_spec[0].shape
         if not input_shape[0]:
-            raise Exception('If a RNN is stateful, a complete ' +
-                            'input_shape must be provided (including batch size).')
+            raise ValueError('If a RNN is stateful, it needs to know '
+                             'its batch size. Specify the batch size '
+                             'of your input tensors: \n'
+                             '- If using a Sequential model, '
+                             'specify the batch size by passing '
+                             'a `batch_input_shape` '
+                             'argument to your first layer.\n'
+                             '- If using the functional API, specify '
+                             'the time dimension by passing a '
+                             '`batch_shape` argument to your Input layer.')
         if hasattr(self, 'states'):
             K.set_value(self.states[0],
                         np.zeros((input_shape[0], self.output_dim)))
@@ -328,7 +363,7 @@ class SimpleRNN(Recurrent):
 
     def preprocess_input(self, x):
         if self.consume_less == 'cpu':
-            input_shape = self.input_spec[0].shape
+            input_shape = K.int_shape(x)
             input_dim = input_shape[2]
             timesteps = input_shape[1]
             return time_distributed_dense(x, self.W, self.b, self.dropout_W,
@@ -360,10 +395,10 @@ class SimpleRNN(Recurrent):
         else:
             constants.append(K.cast_to_floatx(1.))
         if self.consume_less == 'cpu' and 0 < self.dropout_W < 1:
-            input_shape = self.input_spec[0].shape
+            input_shape = K.int_shape(x)
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.tile(ones, (1, input_dim))
+            ones = K.tile(ones, (1, int(input_dim)))
             B_W = K.in_train_phase(K.dropout(ones, self.dropout_W), ones)
             constants.append(B_W)
         else:
@@ -385,7 +420,7 @@ class SimpleRNN(Recurrent):
 
 
 class GRU(Recurrent):
-    '''Gated Recurrent Unit - Cho et al. 2014.
+    """Gated Recurrent Unit - Cho et al. 2014.
 
     # Arguments
         output_dim: dimension of the internal projections and the final output.
@@ -407,10 +442,11 @@ class GRU(Recurrent):
         dropout_U: float between 0 and 1. Fraction of the input units to drop for recurrent connections.
 
     # References
-        - [On the Properties of Neural Machine Translation: Encoder–Decoder Approaches](http://www.aclweb.org/anthology/W14-4012)
-        - [Empirical Evaluation of Gated Recurrent Neural Networks on Sequence Modeling](http://arxiv.org/pdf/1412.3555v1.pdf)
+        - [On the Properties of Neural Machine Translation: Encoder-Decoder Approaches](https://arxiv.org/abs/1409.1259)
+        - [Empirical Evaluation of Gated Recurrent Neural Networks on Sequence Modeling](http://arxiv.org/abs/1412.3555v1)
         - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](http://arxiv.org/abs/1512.05287)
-    '''
+    """
+
     def __init__(self, output_dim,
                  init='glorot_uniform', inner_init='orthogonal',
                  activation='tanh', inner_activation='hard_sigmoid',
@@ -424,7 +460,8 @@ class GRU(Recurrent):
         self.W_regularizer = regularizers.get(W_regularizer)
         self.U_regularizer = regularizers.get(U_regularizer)
         self.b_regularizer = regularizers.get(b_regularizer)
-        self.dropout_W, self.dropout_U = dropout_W, dropout_U
+        self.dropout_W = dropout_W
+        self.dropout_U = dropout_U
 
         if self.dropout_W or self.dropout_U:
             self.uses_learning_phase = True
@@ -441,67 +478,71 @@ class GRU(Recurrent):
             self.states = [None]
 
         if self.consume_less == 'gpu':
-
-            self.W = self.init((self.input_dim, 3 * self.output_dim),
-                               name='{}_W'.format(self.name))
-            self.U = self.inner_init((self.output_dim, 3 * self.output_dim),
-                                     name='{}_U'.format(self.name))
-
-            self.b = K.variable(np.hstack((np.zeros(self.output_dim),
-                                           np.zeros(self.output_dim),
-                                           np.zeros(self.output_dim))),
-                                name='{}_b'.format(self.name))
-
-            self.trainable_weights = [self.W, self.U, self.b]
+            self.W = self.add_weight((self.input_dim, 3 * self.output_dim),
+                                     initializer=self.init,
+                                     name='{}_W'.format(self.name),
+                                     regularizer=self.W_regularizer)
+            self.U = self.add_weight((self.output_dim, 3 * self.output_dim),
+                                     initializer=self.inner_init,
+                                     name='{}_U'.format(self.name),
+                                     regularizer=self.U_regularizer)
+            self.b = self.add_weight((self.output_dim * 3,),
+                                     initializer='zero',
+                                     name='{}_b'.format(self.name),
+                                     regularizer=self.b_regularizer)
         else:
-
-            self.W_z = self.init((self.input_dim, self.output_dim),
-                                 name='{}_W_z'.format(self.name))
-            self.U_z = self.inner_init((self.output_dim, self.output_dim),
-                                       name='{}_U_z'.format(self.name))
-            self.b_z = K.zeros((self.output_dim,), name='{}_b_z'.format(self.name))
-
-            self.W_r = self.init((self.input_dim, self.output_dim),
-                                 name='{}_W_r'.format(self.name))
-            self.U_r = self.inner_init((self.output_dim, self.output_dim),
-                                       name='{}_U_r'.format(self.name))
-            self.b_r = K.zeros((self.output_dim,), name='{}_b_r'.format(self.name))
-
-            self.W_h = self.init((self.input_dim, self.output_dim),
-                                 name='{}_W_h'.format(self.name))
-            self.U_h = self.inner_init((self.output_dim, self.output_dim),
-                                       name='{}_U_h'.format(self.name))
-            self.b_h = K.zeros((self.output_dim,), name='{}_b_h'.format(self.name))
-
-            self.trainable_weights = [self.W_z, self.U_z, self.b_z,
-                                      self.W_r, self.U_r, self.b_r,
-                                      self.W_h, self.U_h, self.b_h]
-
+            self.W_z = self.add_weight((self.input_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_W_z'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.U_z = self.add_weight((self.output_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_U_z'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.b_z = self.add_weight((self.output_dim,),
+                                       initializer='zero',
+                                       name='{}_b_z'.format(self.name),
+                                       regularizer=self.b_regularizer)
+            self.W_r = self.add_weight((self.input_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_W_r'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.U_r = self.add_weight((self.output_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_U_r'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.b_r = self.add_weight((self.output_dim,),
+                                       initializer='zero',
+                                       name='{}_b_r'.format(self.name),
+                                       regularizer=self.b_regularizer)
+            self.W_h = self.add_weight((self.input_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_W_h'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.U_h = self.add_weight((self.output_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_U_h'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.b_h = self.add_weight((self.output_dim,),
+                                       initializer='zero',
+                                       name='{}_b_h'.format(self.name),
+                                       regularizer=self.b_regularizer)
             self.W = K.concatenate([self.W_z, self.W_r, self.W_h])
             self.U = K.concatenate([self.U_z, self.U_r, self.U_h])
             self.b = K.concatenate([self.b_z, self.b_r, self.b_h])
 
-        self.regularizers = []
-        if self.W_regularizer:
-            self.W_regularizer.set_param(self.W)
-            self.regularizers.append(self.W_regularizer)
-        if self.U_regularizer:
-            self.U_regularizer.set_param(self.U)
-            self.regularizers.append(self.U_regularizer)
-        if self.b_regularizer:
-            self.b_regularizer.set_param(self.b)
-            self.regularizers.append(self.b_regularizer)
-
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def reset_states(self):
         assert self.stateful, 'Layer must be stateful.'
         input_shape = self.input_spec[0].shape
         if not input_shape[0]:
-            raise Exception('If a RNN is stateful, a complete ' +
-                            'input_shape must be provided (including batch size).')
+            raise ValueError('If a RNN is stateful, a complete '
+                             'input_shape must be provided '
+                             '(including batch size).')
         if hasattr(self, 'states'):
             K.set_value(self.states[0],
                         np.zeros((input_shape[0], self.output_dim)))
@@ -510,7 +551,7 @@ class GRU(Recurrent):
 
     def preprocess_input(self, x):
         if self.consume_less == 'cpu':
-            input_shape = self.input_spec[0].shape
+            input_shape = K.int_shape(x)
             input_dim = input_shape[2]
             timesteps = input_shape[1]
 
@@ -555,7 +596,7 @@ class GRU(Recurrent):
                 x_r = K.dot(x * B_W[1], self.W_r) + self.b_r
                 x_h = K.dot(x * B_W[2], self.W_h) + self.b_h
             else:
-                raise Exception('Unknown `consume_less` mode.')
+                raise ValueError('Unknown `consume_less` mode.')
             z = self.inner_activation(x_z + K.dot(h_tm1 * B_U[0], self.U_z))
             r = self.inner_activation(x_r + K.dot(h_tm1 * B_U[1], self.U_r))
 
@@ -574,10 +615,10 @@ class GRU(Recurrent):
             constants.append([K.cast_to_floatx(1.) for _ in range(3)])
 
         if 0 < self.dropout_W < 1:
-            input_shape = self.input_spec[0].shape
+            input_shape = K.int_shape(x)
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.tile(ones, (1, input_dim))
+            ones = K.tile(ones, (1, int(input_dim)))
             B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(3)]
             constants.append(B_W)
         else:
@@ -600,7 +641,7 @@ class GRU(Recurrent):
 
 
 class LSTM(Recurrent):
-    '''Long-Short Term Memory unit - Hochreiter 1997.
+    """Long-Short Term Memory unit - Hochreiter 1997.
 
     For a step-by-step description of the algorithm, see
     [this tutorial](http://deeplearning.net/tutorial/lstm.html).
@@ -630,9 +671,10 @@ class LSTM(Recurrent):
     # References
         - [Long short-term memory](http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf) (original 1997 paper)
         - [Learning to forget: Continual prediction with LSTM](http://www.mitpressjournals.org/doi/pdf/10.1162/089976600300015015)
-        - [Supervised sequence labelling with recurrent neural networks](http://www.cs.toronto.edu/~graves/preprint.pdf)
+        - [Supervised sequence labeling with recurrent neural networks](http://www.cs.toronto.edu/~graves/preprint.pdf)
         - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](http://arxiv.org/abs/1512.05287)
-    '''
+    """
+
     def __init__(self, output_dim,
                  init='glorot_uniform', inner_init='orthogonal',
                  forget_bias_init='one', activation='tanh',
@@ -648,7 +690,8 @@ class LSTM(Recurrent):
         self.W_regularizer = regularizers.get(W_regularizer)
         self.U_regularizer = regularizers.get(U_regularizer)
         self.b_regularizer = regularizers.get(b_regularizer)
-        self.dropout_W, self.dropout_U = dropout_W, dropout_U
+        self.dropout_W = dropout_W
+        self.dropout_U = dropout_U
 
         if self.dropout_W or self.dropout_U:
             self.uses_learning_phase = True
@@ -665,73 +708,94 @@ class LSTM(Recurrent):
             self.states = [None, None]
 
         if self.consume_less == 'gpu':
-            self.W = self.init((self.input_dim, 4 * self.output_dim),
-                               name='{}_W'.format(self.name))
-            self.U = self.inner_init((self.output_dim, 4 * self.output_dim),
-                                     name='{}_U'.format(self.name))
+            self.W = self.add_weight((self.input_dim, 4 * self.output_dim),
+                                     initializer=self.init,
+                                     name='{}_W'.format(self.name),
+                                     regularizer=self.W_regularizer)
+            self.U = self.add_weight((self.output_dim, 4 * self.output_dim),
+                                     initializer=self.inner_init,
+                                     name='{}_U'.format(self.name),
+                                     regularizer=self.U_regularizer)
 
-            self.b = K.variable(np.hstack((np.zeros(self.output_dim),
-                                           K.get_value(self.forget_bias_init((self.output_dim,))),
-                                           np.zeros(self.output_dim),
-                                           np.zeros(self.output_dim))),
-                                name='{}_b'.format(self.name))
-            self.trainable_weights = [self.W, self.U, self.b]
+            def b_reg(shape, name=None):
+                return K.variable(np.hstack((np.zeros(self.output_dim),
+                                             K.get_value(self.forget_bias_init((self.output_dim,))),
+                                             np.zeros(self.output_dim),
+                                             np.zeros(self.output_dim))),
+                                  name='{}_b'.format(self.name))
+            self.b = self.add_weight((self.output_dim * 4,),
+                                     initializer=b_reg,
+                                     name='{}_b'.format(self.name),
+                                     regularizer=self.b_regularizer)
         else:
-            self.W_i = self.init((self.input_dim, self.output_dim),
-                                 name='{}_W_i'.format(self.name))
-            self.U_i = self.inner_init((self.output_dim, self.output_dim),
-                                       name='{}_U_i'.format(self.name))
-            self.b_i = K.zeros((self.output_dim,), name='{}_b_i'.format(self.name))
-
-            self.W_f = self.init((self.input_dim, self.output_dim),
-                                 name='{}_W_f'.format(self.name))
-            self.U_f = self.inner_init((self.output_dim, self.output_dim),
-                                       name='{}_U_f'.format(self.name))
-            self.b_f = self.forget_bias_init((self.output_dim,),
-                                             name='{}_b_f'.format(self.name))
-
-            self.W_c = self.init((self.input_dim, self.output_dim),
-                                 name='{}_W_c'.format(self.name))
-            self.U_c = self.inner_init((self.output_dim, self.output_dim),
-                                       name='{}_U_c'.format(self.name))
-            self.b_c = K.zeros((self.output_dim,), name='{}_b_c'.format(self.name))
-
-            self.W_o = self.init((self.input_dim, self.output_dim),
-                                 name='{}_W_o'.format(self.name))
-            self.U_o = self.inner_init((self.output_dim, self.output_dim),
-                                       name='{}_U_o'.format(self.name))
-            self.b_o = K.zeros((self.output_dim,), name='{}_b_o'.format(self.name))
+            self.W_i = self.add_weight((self.input_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_W_i'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.U_i = self.add_weight((self.output_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_U_i'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.b_i = self.add_weight((self.output_dim,),
+                                       initializer='zero',
+                                       name='{}_b_i'.format(self.name),
+                                       regularizer=self.b_regularizer)
+            self.W_f = self.add_weight((self.input_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_W_f'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.U_f = self.add_weight((self.output_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_U_f'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.b_f = self.add_weight((self.output_dim,),
+                                       initializer=self.forget_bias_init,
+                                       name='{}_b_f'.format(self.name),
+                                       regularizer=self.b_regularizer)
+            self.W_c = self.add_weight((self.input_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_W_c'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.U_c = self.add_weight((self.output_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_U_c'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.b_c = self.add_weight((self.output_dim,),
+                                       initializer='zero',
+                                       name='{}_b_c'.format(self.name),
+                                       regularizer=self.b_regularizer)
+            self.W_o = self.add_weight((self.input_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_W_o'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.U_o = self.add_weight((self.output_dim, self.output_dim),
+                                       initializer=self.init,
+                                       name='{}_U_o'.format(self.name),
+                                       regularizer=self.W_regularizer)
+            self.b_o = self.add_weight((self.output_dim,),
+                                       initializer='zero',
+                                       name='{}_b_o'.format(self.name),
+                                       regularizer=self.b_regularizer)
 
             self.trainable_weights = [self.W_i, self.U_i, self.b_i,
                                       self.W_c, self.U_c, self.b_c,
                                       self.W_f, self.U_f, self.b_f,
                                       self.W_o, self.U_o, self.b_o]
-
             self.W = K.concatenate([self.W_i, self.W_f, self.W_c, self.W_o])
             self.U = K.concatenate([self.U_i, self.U_f, self.U_c, self.U_o])
             self.b = K.concatenate([self.b_i, self.b_f, self.b_c, self.b_o])
 
-        self.regularizers = []
-        if self.W_regularizer:
-            self.W_regularizer.set_param(self.W)
-            self.regularizers.append(self.W_regularizer)
-        if self.U_regularizer:
-            self.U_regularizer.set_param(self.U)
-            self.regularizers.append(self.U_regularizer)
-        if self.b_regularizer:
-            self.b_regularizer.set_param(self.b)
-            self.regularizers.append(self.b_regularizer)
-
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def reset_states(self):
         assert self.stateful, 'Layer must be stateful.'
         input_shape = self.input_spec[0].shape
         if not input_shape[0]:
-            raise Exception('If a RNN is stateful, a complete ' +
-                            'input_shape must be provided (including batch size).')
+            raise ValueError('If a RNN is stateful, a complete ' +
+                             'input_shape must be provided (including batch size).')
         if hasattr(self, 'states'):
             K.set_value(self.states[0],
                         np.zeros((input_shape[0], self.output_dim)))
@@ -747,7 +811,7 @@ class LSTM(Recurrent):
                 dropout = self.dropout_W
             else:
                 dropout = 0
-            input_shape = self.input_spec[0].shape
+            input_shape = K.int_shape(x)
             input_dim = input_shape[2]
             timesteps = input_shape[1]
 
@@ -793,7 +857,7 @@ class LSTM(Recurrent):
                 x_c = K.dot(x * B_W[2], self.W_c) + self.b_c
                 x_o = K.dot(x * B_W[3], self.W_o) + self.b_o
             else:
-                raise Exception('Unknown `consume_less` mode.')
+                raise ValueError('Unknown `consume_less` mode.')
 
             i = self.inner_activation(x_i + K.dot(h_tm1 * B_U[0], self.U_i))
             f = self.inner_activation(x_f + K.dot(h_tm1 * B_U[1], self.U_f))
@@ -814,10 +878,10 @@ class LSTM(Recurrent):
             constants.append([K.cast_to_floatx(1.) for _ in range(4)])
 
         if 0 < self.dropout_W < 1:
-            input_shape = self.input_spec[0].shape
+            input_shape = K.int_shape(x)
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.tile(ones, (1, input_dim))
+            ones = K.tile(ones, (1, int(input_dim)))
             B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(4)]
             constants.append(B_W)
         else:

keras/layers/wrappers.py

@@ -1,8 +1,12 @@
-from ..engine import Layer, InputSpec
+import copy
+from ..engine import Layer
+from ..engine import InputSpec
 from .. import backend as K
 
 
 class Wrapper(Layer):
+    """Abstract wrapper base class.
+    """
 
     def __init__(self, layer, **kwargs):
         self.layer = layer
@@ -10,23 +14,14 @@ class Wrapper(Layer):
         super(Wrapper, self).__init__(**kwargs)
 
     def build(self, input_shape=None):
-        '''Assumes that self.layer is already set.
-        Should be called at the end of .build() in the
-        children classes.
-        '''
+        # Assumes that self.layer is already set.
+        # Should be called at the end of .build() in the children classes.
         self.trainable_weights = getattr(self.layer, 'trainable_weights', [])
         self.non_trainable_weights = getattr(self.layer, 'non_trainable_weights', [])
         self.updates = getattr(self.layer, 'updates', [])
-        self.regularizers = getattr(self.layer, 'regularizers', [])
+        self.losses = getattr(self.layer, 'losses', [])
         self.constraints = getattr(self.layer, 'constraints', {})
 
-        # properly attribute the current layer to
-        # regularizers that need access to it
-        # (e.g. ActivityRegularizer).
-        for regularizer in self.regularizers:
-            if hasattr(regularizer, 'set_layer'):
-                regularizer.set_layer(self)
-
     def get_weights(self):
         weights = self.layer.get_weights()
         return weights
@@ -48,18 +43,19 @@ class Wrapper(Layer):
 
 
 class TimeDistributed(Wrapper):
-    """This wrapper allows to apply a layer to every
-    temporal slice of an input.
+    """This wrapper allows to apply a layer to every temporal slice of an input.
 
-    The input should be at least 3D,
-    and the dimension of index one will be considered to be
-    the temporal dimension.
+    The input should be at least 3D, and the dimension of index one
+    will be considered to be the temporal dimension.
 
-    Consider a batch of 32 samples, where each sample is a sequence of 10
-    vectors of 16 dimensions. The batch input shape of the layer is then `(32, 10, 16)`
-    (and the `input_shape`, not including the samples dimension, is `(10, 16)`).
+    Consider a batch of 32 samples,
+    where each sample is a sequence of 10 vectors of 16 dimensions.
+    The batch input shape of the layer is then `(32, 10, 16)`,
+    and the `input_shape`, not including the samples dimension, is `(10, 16)`.
+
+    You can then use `TimeDistributed` to apply a `Dense` layer
+    to each of the 10 timesteps, independently:
 
-    You can then use `TimeDistributed` to apply a `Dense` layer to each of the 10 timesteps, independently:
     ```python
         # as the first layer in a model
         model = Sequential()
@@ -73,19 +69,19 @@ class TimeDistributed(Wrapper):
 
     The output will then have shape `(32, 10, 8)`.
 
-    Note this is strictly equivalent to using `layers.core.TimeDistributedDense`.
-    However what is different about `TimeDistributed`
-    is that it can be used with arbitrary layers, not just `Dense`,
+    `TimeDistributed` can be used with arbitrary layers, not just `Dense`,
     for instance with a `Convolution2D` layer:
 
     ```python
         model = Sequential()
-        model.add(TimeDistributed(Convolution2D(64, 3, 3), input_shape=(10, 3, 299, 299)))
+        model.add(TimeDistributed(Convolution2D(64, 3, 3),
+                                  input_shape=(10, 3, 299, 299)))
     ```
 
     # Arguments
         layer: a layer instance.
     """
+
     def __init__(self, layer, **kwargs):
         self.supports_masking = True
         super(TimeDistributed, self).__init__(layer, **kwargs)
@@ -105,16 +101,18 @@ class TimeDistributed(Wrapper):
         timesteps = input_shape[1]
         return (child_output_shape[0], timesteps) + child_output_shape[1:]
 
-    def call(self, X, mask=None):
-        input_shape = self.input_spec[0].shape
+    def call(self, inputs, mask=None):
+        input_shape = K.int_shape(inputs)
         if input_shape[0]:
             # batch size matters, use rnn-based implementation
-            def step(x, states):
+            def step(x, _):
                 output = self.layer.call(x)
                 return output, []
 
-            last_output, outputs, states = K.rnn(step, X,
-                                                 initial_states=[])
+            _, outputs, _ = K.rnn(step, inputs,
+                                  initial_states=[],
+                                  input_length=input_shape[1],
+                                  unroll=False)
             y = outputs
         else:
             # no batch size specified, therefore the layer will be able
@@ -122,19 +120,26 @@ class TimeDistributed(Wrapper):
             # we can go with reshape-based implementation for performance
             input_length = input_shape[1]
             if not input_length:
-                input_length = K.shape(X)[1]
-            X = K.reshape(X, (-1, ) + input_shape[2:])  # (nb_samples * timesteps, ...)
-            y = self.layer.call(X)  # (nb_samples * timesteps, ...)
+                input_length = K.shape(inputs)[1]
+            # (nb_samples * timesteps, ...)
+            inputs = K.reshape(inputs, (-1,) + input_shape[2:])
+            y = self.layer.call(inputs)  # (nb_samples * timesteps, ...)
             # (nb_samples, timesteps, ...)
             output_shape = self.get_output_shape_for(input_shape)
             y = K.reshape(y, (-1, input_length) + output_shape[2:])
+
+        # Apply activity regularizer if any:
+        if (hasattr(self.layer, 'activity_regularizer') and
+           self.layer.activity_regularizer is not None):
+            regularization_loss = self.layer.activity_regularizer(y)
+            self.add_loss(regularization_loss, inputs)
         return y
 
 
 class Bidirectional(Wrapper):
-    ''' Bidirectional wrapper for RNNs.
+    """Bidirectional wrapper for RNNs.
 
-    # Arguments:
+    # Arguments
         layer: `Recurrent` instance.
         merge_mode: Mode by which outputs of the
             forward and backward RNNs will be combined.
@@ -142,7 +147,7 @@ class Bidirectional(Wrapper):
             If None, the outputs will not be combined,
             they will be returned as a list.
 
-    # Examples:
+    # Examples
 
     ```python
         model = Sequential()
@@ -152,13 +157,14 @@ class Bidirectional(Wrapper):
         model.add(Activation('softmax'))
         model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
     ```
-    '''
+    """
+
     def __init__(self, layer, merge_mode='concat', weights=None, **kwargs):
         if merge_mode not in ['sum', 'mul', 'ave', 'concat', None]:
             raise ValueError('Invalid merge mode. '
                              'Merge mode should be one of '
                              '{"sum", "mul", "ave", "concat", None}')
-        self.forward_layer = layer
+        self.forward_layer = copy.copy(layer)
         config = layer.get_config()
         config['go_backwards'] = not config['go_backwards']
         self.backward_layer = layer.__class__.from_config(config)
@@ -192,21 +198,21 @@ class Bidirectional(Wrapper):
         elif self.merge_mode is None:
             return [self.forward_layer.get_output_shape_for(input_shape)] * 2
 
-    def call(self, X, mask=None):
-        Y = self.forward_layer.call(X, mask)
-        Y_rev = self.backward_layer.call(X, mask)
+    def call(self, inputs, mask=None):
+        y = self.forward_layer.call(inputs, mask)
+        y_rev = self.backward_layer.call(inputs, mask)
         if self.return_sequences:
-            Y_rev = K.reverse(Y_rev, 1)
+            y_rev = K.reverse(y_rev, 1)
         if self.merge_mode == 'concat':
-            return K.concatenate([Y, Y_rev])
+            return K.concatenate([y, y_rev])
         elif self.merge_mode == 'sum':
-            return Y + Y_rev
+            return y + y_rev
         elif self.merge_mode == 'ave':
-            return (Y + Y_rev) / 2
+            return (y + y_rev) / 2
         elif self.merge_mode == 'mul':
-            return Y * Y_rev
+            return y * y_rev
         elif self.merge_mode is None:
-            return [Y, Y_rev]
+            return [y, y_rev]
 
     def reset_states(self):
         self.forward_layer.reset_states()
@@ -228,13 +234,15 @@ class Bidirectional(Wrapper):
     @property
     def trainable_weights(self):
         if hasattr(self.forward_layer, 'trainable_weights'):
-            return self.forward_layer.trainable_weights + self.backward_layer.trainable_weights
+            return (self.forward_layer.trainable_weights +
+                    self.backward_layer.trainable_weights)
         return []
 
     @property
     def non_trainable_weights(self):
         if hasattr(self.forward_layer, 'non_trainable_weights'):
-            return self.forward_layer.non_trainable_weights + self.backward_layer.non_trainable_weights
+            return (self.forward_layer.non_trainable_weights +
+                    self.backward_layer.non_trainable_weights)
         return []
 
     @property
@@ -244,18 +252,18 @@ class Bidirectional(Wrapper):
         return []
 
     @property
-    def regularizers(self):
-        if hasattr(self.forward_layer, 'regularizers'):
-            return self.forward_layer.regularizers + self.backward_layer.regularizers
+    def losses(self):
+        if hasattr(self.forward_layer, 'losses'):
+            return self.forward_layer.losses + self.backward_layer.losses
         return []
 
     @property
     def constraints(self):
-        _constraints = {}
+        constraints = {}
         if hasattr(self.forward_layer, 'constraints'):
-            _constraints.update(self.forward_layer.constraints)
-            _constraints.update(self.backward_layer.constraints)
-        return _constraints
+            constraints.update(self.forward_layer.constraints)
+            constraints.update(self.backward_layer.constraints)
+        return constraints
 
     def get_config(self):
         config = {"merge_mode": self.merge_mode}

keras/legacy/models.py

@@ -1,775 +0,0 @@
-from collections import OrderedDict
-import warnings
-import copy
-
-from .. import backend as K
-from ..layers import InputLayer, Layer, Merge
-from ..engine.training import Model
-
-
-class Graph(Model):
-    '''Arbitrary connection graph.
-
-    THIS IS A LEGACY MODEL AND SHOULD NOT BE USED
-    except for backwards compatibility support.
-
-    For multi-inputs/multi-outputs models, or
-    models using shared layers, use the functional API instead.
-    '''
-
-    def __init__(self, name=None):
-        # model attributes
-        self.inbound_nodes = []
-        self.outbound_nodes = []
-        self.built = False
-        self.supports_masking = False
-
-        # legacy attributes (we prefix them with _graph_)
-        self._graph_namespace = set()  # strings
-        self._graph_nodes = OrderedDict()  # layer-like
-        self._graph_inputs = OrderedDict()  # layer-like
-        self._graph_outputs = OrderedDict()  # layer-like
-        self._graph_input_config = []  # dicts
-        self._graph_output_config = []  # dicts
-        self._graph_node_config = []  # dicts
-        self._graph_shared_nodes_names = []
-
-        if not name:
-            prefix = 'graph_'
-            name = prefix + str(K.get_uid(prefix))
-        self.name = name
-
-    def __call__(self, x, mask=None):
-        self.build()
-        return super(Graph, self).__call__(x, mask)
-
-    def build(self, input_shape=None):
-        # this will crash if the input/output layers have multiple nodes
-        # no plans to support that case since Graph is deprecated
-        input_tensors = [layer.output for layer in self._graph_inputs.values()]
-        output_tensors = [layer.output for layer in self._graph_outputs.values()]
-        # actually create the model
-        super(Graph, self).__init__(input_tensors,
-                                    output_tensors,
-                                    name=self.name)
-        self.built = True
-
-    def compile(self, optimizer, loss,
-                metrics=[],
-                sample_weight_modes=None,
-                loss_weights=None,
-                **kwargs):
-        '''Configures the learning process.
-
-        # Arguments
-            optimizer: str (name of optimizer) or optimizer object.
-                See [optimizers](optimizers.md).
-            loss: dictionary mapping the name(s) of the output(s) to
-                a loss function (string name of objective function or
-                objective function. See [objectives](objectives.md)).
-            metrics: list of str (name of metrics) or
-                list of metrics functions. See [metrics](metrics.md).
-            sample_weight_modes: optional dictionary mapping certain
-                output names to a sample weight mode ("temporal" and None
-                are the only supported modes). If you need to do
-                timestep-wise loss weighting on one of your graph outputs,
-                you will need to set the sample weight mode for this output
-                to "temporal".
-            loss_weights: dictionary you can pass to specify a weight
-                coefficient for each loss function (in a multi-output model).
-                If no loss weight is specified for an output,
-                the weight for this output's loss will be considered to be 1.
-            kwargs: for Theano backend, these are passed into K.function.
-                Ignored for Tensorflow backend.
-        '''
-        # create the underlying Model
-        if not self.built:
-            self.build()
-        super(Graph, self).compile(optimizer, loss,
-                                   metrics=metrics,
-                                   sample_weight_mode=sample_weight_modes,
-                                   loss_weights=loss_weights,
-                                   **kwargs)
-
-    def add_input(self, name, input_shape=None,
-                  batch_input_shape=None, dtype='float'):
-        '''Adds an input to the graph.
-
-        # Arguments:
-            name: string. The name of the new input.
-                Must be unique in the graph.
-            input_shape: a tuple of integers,
-                the expected shape of the input samples.
-                Does not include the batch size.
-            batch_input_shape: a tuple of integers,
-                the expected shape of the whole input batch,
-                including the batch size.
-            dtype: 'float', or 'int'.
-        '''
-        if name in self._graph_namespace:
-            raise Exception('Duplicate node identifier: ' + name)
-        self._graph_namespace.add(name)
-        self.built = False
-
-        if dtype[:3] == 'int':
-            dtype = 'int32'
-        elif dtype[:5] == 'float':
-            dtype = K.floatx()
-        else:
-            raise Exception('Uknown dtype (should be "int" or "float"): ' +
-                            str(dtype))
-
-        # create input layer
-        input_layer = InputLayer(input_shape=input_shape,
-                                 batch_input_shape=batch_input_shape,
-                                 name=name, input_dtype=dtype)
-        self._graph_inputs[name] = input_layer
-
-        # append input config to self._graph_input_config
-        config = {'name': name, 'dtype': dtype}
-        if batch_input_shape:
-            config['batch_input_shape'] = batch_input_shape
-        else:
-            config['input_shape'] = input_shape
-        self._graph_input_config.append(config)
-
-    def add_node(self, layer, name, input=None, inputs=[],
-                 merge_mode='concat', concat_axis=-1, dot_axes=-1,
-                 create_output=False):
-        '''Adds a node in the graph. It can be connected to multiple
-        inputs, which will first be merged into one tensor
-        according to the mode specified.
-
-        # Arguments
-            layer: the layer at the node.
-            name: name for the node.
-            input: when connecting the layer to a single input,
-                this is the name of the incoming node.
-            inputs: when connecting the layer to multiple inputs,
-                this is a list of names of incoming nodes.
-            merge_mode: one of {concat, sum, dot, ave, mul}
-            concat_axis: when `merge_mode=='concat'`, this is the
-                input concatenation axis.
-            dot_axes: when `merge_mode='dot'`,
-                this is the contraction axes specification;
-                see the `Merge` layer for details.
-            create_output: boolean. Set this to `True` if you want the output
-                of your node to be an output of the graph.
-        '''
-        if name in self._graph_namespace:
-            raise Exception('Duplicate node identifier: ' + name)
-        self._graph_namespace.add(name)
-        layer.name = name
-        self.built = False
-
-        if input:
-            if input not in self._graph_namespace:
-                raise Exception('Unknown node/input identifier: ' + input)
-            if input in self._graph_nodes:
-                layer.add_inbound_node(self._graph_nodes[input])
-            elif input in self._graph_inputs:
-                layer.add_inbound_node(self._graph_inputs[input])
-        if inputs:
-            to_merge = []
-            for n in inputs:
-                if n in self._graph_nodes:
-                    to_merge.append(self._graph_nodes[n])
-                elif n in self._graph_inputs:
-                    to_merge.append(self._graph_inputs[n])
-                else:
-                    raise Exception('Unknown identifier: ' + n)
-            merge = Merge(to_merge, mode=merge_mode,
-                          concat_axis=concat_axis, dot_axes=dot_axes,
-                          name='merge_inputs_for_' + name)
-            layer.add_inbound_node(merge)
-        self._graph_nodes[name] = layer
-        self._graph_node_config.append({'name': name,
-                                        'input': input,
-                                        'inputs': inputs,
-                                        'merge_mode': merge_mode,
-                                        'concat_axis': concat_axis,
-                                        'dot_axes': dot_axes,
-                                        'create_output': create_output})
-        if create_output:
-            self.add_output(name, input=name)
-
-    def add_shared_node(self, layer, name, inputs=[], merge_mode=None,
-                        concat_axis=-1, dot_axes=-1, outputs=[],
-                        create_output=False):
-        '''Used to share a same layer across multiple nodes.
-
-        Supposed, for instance, that you want to apply one same `Dense` layer
-        after two different nodes ('node_a' and 'node_b').
-        You can then add the dense layer as a shared node by calling:
-
-        ```python
-        model.add_shared_node(my_dense, name='shared_dense', inputs=['node_a', 'node_b'], ...)
-        ```
-
-        If you want access to the output of dense(node_a) and dense(node_b) separately,
-        you can add these outputs to the Graph by passing an `outputs` argument:
-
-        ```python
-        model.add_shared_node(my_dense, name='shared_dense', inputs=['node_a', 'node_b'],
-                              outputs=['dense_output_a', 'dense_outputs_b'])
-        ```
-
-        Otherwise you can merge these different outputs via `merge_mode`.
-        In that case you can access the merged output
-        under the identifier `name`.
-
-        # Arguments
-            layer: The layer to be shared across multiple inputs
-            name: Name of the shared node
-            inputs: List of names of input nodes
-            merge_mode: Same meaning as `merge_mode` argument of `add_node()`
-            concat_axis: Same meaning as `concat_axis` argument of `add_node()`
-            dot_axes: Same meaning as `dot_axes` argument of `add_node()`
-            outputs: Used when `merge_mode=None`. Names for the output nodes.
-            create_output: Same meaning as `create_output` argument of `add_node()`.
-        '''
-        if name in self._graph_namespace:
-            raise Exception('Duplicate node identifier: ' + name)
-        self._graph_namespace.add(name)
-        self.built = False
-
-        for o in outputs:
-            if o in self._graph_namespace:
-                raise Exception('Duplicate node identifier: ' + o)
-        if merge_mode:
-            if merge_mode not in {'sum', 'ave', 'mul', 'dot', 'cos', 'concat'}:
-                raise Exception('Invalid merge mode:', merge_mode)
-        input_layers = []
-        for i in range(len(inputs)):
-            input = inputs[i]
-            if input in self._graph_nodes:
-                n = self._graph_nodes[input]
-                input_layers.append(n)
-            elif input in self._graph_inputs:
-                n = self._graph_inputs[input]
-                input_layers.append(n)
-            else:
-                raise Exception('Unknown identifier: ' + input)
-
-        created_node_indices = []
-        for input_layer in input_layers:
-            created_node_indices.append(len(layer.inbound_nodes))
-            layer.add_inbound_node(input_layer)
-
-        if merge_mode:
-            layer.name = 'input_for_' + name
-            # collect all output nodes of layer and merge them into a single output
-            merge = Merge([layer for _ in range(len(inputs))],
-                          mode=merge_mode,
-                          concat_axis=concat_axis, dot_axes=dot_axes,
-                          node_indices=created_node_indices,
-                          name=name)
-            self._graph_nodes[name] = merge
-            if create_output:
-                self.add_output(name, input=name)
-        else:
-            layer.name = name
-            # create one new layer per output node of layer,
-            # and add them to the Graph with their own identifiers
-            if len(outputs) != len(inputs):
-                raise Exception('When using merge_mode=None, '
-                                'you should provide a list of '
-                                'output names (`output` argument) '
-                                'the same size as `input`.')
-            for i in range(len(outputs)):
-                output_layer_name = outputs[i]
-                output_layer = Layer(name=output_layer_name)
-                output_layer.add_inbound_node(layer, created_node_indices[i])
-                self._graph_namespace.add(output_layer_name)
-                self._graph_nodes[output_layer_name] = output_layer
-                if create_output:
-                    self.add_output(output_layer_name, input=output_layer_name)
-
-        self._graph_node_config.append({'name': name,
-                                        'layer': {
-                                            'config': layer.get_config(),
-                                            'class_name': layer.__class__.__name__,
-                                        },
-                                        'inputs': inputs,
-                                        'merge_mode': merge_mode,
-                                        'concat_axis': concat_axis,
-                                        'dot_axes': dot_axes,
-                                        'outputs': outputs,
-                                        'create_output': create_output if merge_mode else False})
-        self._graph_shared_nodes_names.append(name)
-
-    def add_output(self, name, input=None, inputs=[],
-                   merge_mode='concat', concat_axis=-1, dot_axes=-1):
-        '''Adds an output to the graph.
-
-        This output can merge several node outputs into a single output.
-
-        # Arguments
-            name: name of the output.
-            input: when connecting the layer to a single input,
-                this is the name of the incoming node.
-            inputs: when connecting the layer to multiple inputs,
-                this is a list of names of incoming nodes.
-            merge_mode: one of {concat, sum, dot, ave, mul}
-            concat_axis: when `merge_mode=='concat'`, this is the
-                input concatenation axis.
-            dot_axes: when `merge_mode='dot'`,
-                this is the contraction axes specification;
-                see the `Merge layer for details.
-        '''
-        if name not in self._graph_namespace:
-            self._graph_namespace.add(name)
-        if name in self._graph_outputs:
-            raise Exception('Duplicate output identifier:', name)
-        self.built = False
-
-        if input:
-            if input in self._graph_nodes:
-                layer = self._graph_nodes[input]
-            elif input in self._graph_inputs:
-                layer = self._graph_inputs[input]
-            else:
-                raise Exception('Unknown node/input identifier: ' + input)
-            if layer.name == name:
-                self._graph_outputs[name] = layer
-            else:
-                layer.name = name
-                self._graph_outputs[name] = layer
-        if inputs:
-            to_merge = []
-            for n in inputs:
-                if n not in self._graph_nodes:
-                    raise Exception('Unknown identifier: ' + n)
-                to_merge.append(self._graph_nodes[n])
-            merge = Merge(to_merge, mode=merge_mode,
-                          concat_axis=concat_axis, dot_axes=dot_axes,
-                          name=name)
-            self._graph_outputs[name] = merge
-
-        self._graph_output_config.append({'name': name,
-                                          'input': input,
-                                          'inputs': inputs,
-                                          'merge_mode': merge_mode,
-                                          'concat_axis': concat_axis,
-                                          'dot_axes': dot_axes})
-
-    def _get_x(self, data):
-        x = []
-        for key in self._graph_inputs.keys():
-            if key not in data:
-                raise Exception('Expected to be provided an array '
-                                '(in dict argument `data`) for input "' +
-                                key + '".')
-            x.append(data[key])
-        return x
-
-    def _get_y(self, data):
-        y = []
-        for key in self._graph_outputs.keys():
-            if key not in data:
-                raise Exception('Expected to be provided an array '
-                                '(in dict argument `data`) for output "' +
-                                key + '".')
-            y.append(data[key])
-        return y
-
-    def fit(self, data, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
-            validation_split=0., validation_data=None, shuffle=True,
-            class_weight=None, sample_weight=None, **kwargs):
-        '''Trains the model for a fixed number of epochs.
-
-        Returns a history object. Its `history` attribute is a record of
-        training loss values at successive epochs,
-        as well as validation loss values (if applicable).
-
-        # Arguments
-            data: dictionary mapping input names and outputs names to
-                appropriate Numpy arrays. All arrays should contain
-                the same number of samples.
-            batch_size: int. Number of samples per gradient update.
-            nb_epoch: int.
-            verbose: 0 for no logging to stdout,
-                1 for progress bar logging, 2 for one log line per epoch.
-            callbacks: `keras.callbacks.Callback` list. List of callbacks
-                to apply during training. See [callbacks](callbacks.md).
-            validation_split: float (0. < x < 1). Fraction of the data to
-                use as held-out validation data.
-            validation_data: dictionary mapping input names and outputs names
-                to appropriate Numpy arrays to be used as
-                held-out validation data.
-                All arrays should contain the same number of samples.
-                Will override validation_split.
-            shuffle: boolean. Whether to shuffle the samples at each epoch.
-            class_weight: dictionary mapping output names to
-                class weight dictionaries.
-            sample_weight: dictionary mapping output names to
-                numpy arrays of sample weights.
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-
-        if type(validation_data) is tuple:
-            raise Exception('Cannot used sample_weight with '
-                            'validation data with legacy Graph model. '
-                            'validation_data should be a dictionary.')
-        if validation_data:
-            val_x = self._get_x(validation_data)
-            val_y = self._get_y(validation_data)
-            validation_data = (val_x, val_y)
-        return super(Graph, self).fit(x, y,
-                                      batch_size=batch_size,
-                                      nb_epoch=nb_epoch,
-                                      verbose=verbose,
-                                      callbacks=callbacks,
-                                      validation_split=validation_split,
-                                      validation_data=validation_data,
-                                      shuffle=shuffle,
-                                      class_weight=class_weight,
-                                      sample_weight=sample_weight)
-
-    def evaluate(self, data, batch_size=128,
-                 verbose=0, sample_weight={}, **kwargs):
-        '''Computes the loss on some input data, batch by batch.
-
-        Returns the scalar test loss over the data,
-        or a list of metrics values (starting with the test loss)
-        if applicable.
-
-        Arguments: see `fit` method.
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-        return super(Graph, self).evaluate(x, y,
-                                           batch_size=batch_size,
-                                           verbose=verbose,
-                                           sample_weight=sample_weight)
-
-    def predict(self, data, batch_size=128, verbose=0):
-        '''Generates output predictions for the input samples
-        batch by batch.
-
-        Arguments: see `fit` method.
-        '''
-        x = self._get_x(data)
-        output_list = super(Graph, self).predict(x, batch_size=batch_size,
-                                                 verbose=verbose)
-        if not isinstance(output_list, list):
-            output_list = [output_list]
-        return dict(zip(self._graph_outputs, output_list))
-
-    def train_on_batch(self, data,
-                       class_weight={},
-                       sample_weight={}, **kwargs):
-        '''Single gradient update on a batch of samples.
-
-        Returns the scalar train loss over the data,
-        or a list of metrics values (starting with the test loss)
-        if applicable.
-
-        Arguments: see `fit` method.
-        '''
-        if 'accuracy' in kwargs:
-            kwargs.pop('accuracy')
-            warnings.warn('The "accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-        return super(Graph, self).train_on_batch(x, y,
-                                                 sample_weight=sample_weight,
-                                                 class_weight=class_weight)
-
-    def test_on_batch(self, data, sample_weight={}, **kwargs):
-        '''Test the network on a single batch of samples.
-
-        Returns the scalar test loss over the data,
-        or a list of metrics values (starting with the test loss)
-        if applicable.
-
-        Arguments: see `fit` method.
-        '''
-        if 'accuracy' in kwargs:
-            kwargs.pop('accuracy')
-            warnings.warn('The "accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-        return super(Graph, self).test_on_batch(x, y,
-                                                sample_weight=sample_weight)
-
-    def predict_on_batch(self, data):
-        output_list = super(Graph, self).predict_on_batch(data)
-        if not isinstance(output_list, list):
-            output_list = [output_list]
-        return dict(zip(self._graph_outputs, output_list))
-
-    def fit_generator(self, generator, samples_per_epoch, nb_epoch,
-                      verbose=1, callbacks=[],
-                      validation_data=None, nb_val_samples=None,
-                      class_weight={},
-                      max_q_size=10, **kwargs):
-        '''Fits a model on data generated batch-by-batch by a Python generator.
-        The generator is run in parallel to the model, for efficiency.
-        For instance, this allows you to do real-time data augmentation
-        on images on CPU in parallel to training your model on GPU.
-
-        # Arguments
-            generator: a generator.
-                The output of the generator must be either a tuple
-                of dictionaries `(input_data, sample_weight)`
-                or a dictionary `input_data`
-                (mapping names of inputs and outputs to Numpy arrays).
-                All arrays should contain the same number of samples.
-                The generator is expected to loop over its data
-                indefinitely. An epoch finishes when `samples_per_epoch`
-                samples have been seen by the model.
-            samples_per_epoch: integer, number of samples to process before
-                going to the next epoch.
-            nb_epoch: integer, total number of iterations on the data.
-            verbose: verbosity mode, 0, 1, or 2.
-            callbacks: list of callbacks to be called during training.
-            validation_data: dictionary mapping input names and outputs names
-                to appropriate Numpy arrays to be used as
-                held-out validation data, or a generator yielding such
-                dictionaries. All arrays should contain the same number
-                of samples. If a generator, will be called until more than
-                `nb_val_samples` examples have been generated at the
-                end of every epoch. These examples will then be used
-                as the validation data.
-            nb_val_samples: number of samples to use from validation
-                generator at the end of every epoch.
-            class_weight: dictionary mapping class indices to a weight
-                for the class.
-
-        # Returns
-            A `History` object.
-
-        # Examples
-
-        ```python
-            def generate_arrays_from_file(path):
-                while 1:
-                    f = open(path)
-                    for line in f:
-                        # create Numpy arrays of input data
-                        # and labels, from each line in the file
-                        x1, x2, y = process_line(line)
-                        yield ({'input_1': x1, 'input_2': x2, 'output': y})
-                    f.close()
-
-            graph.fit_generator(generate_arrays_from_file('/my_file.txt'),
-                                samples_per_epoch=10000, nb_epoch=10)
-        ```
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if 'nb_worker' in kwargs:
-            kwargs.pop('nb_worker')
-            warnings.warn('The "nb_worker" argument is deprecated, '
-                          'please remove it from your code.')
-        if 'nb_val_worker' in kwargs:
-            kwargs.pop('nb_val_worker')
-            warnings.warn('The "nb_val_worker" argument is deprecated, '
-                          'please remove it from your code.')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-
-        self._train_on_batch = self.train_on_batch
-        self.train_on_batch = super(Graph, self).train_on_batch
-        self._evaluate = self.evaluate
-        self.evaluate = super(Graph, self).evaluate
-
-        if validation_data and type(validation_data) is tuple:
-            raise Exception('Cannot use sample_weight with '
-                            'validation_data in legacy Graph model.')
-        if validation_data and type(validation_data) is dict:
-            validation_data = (self._get_x(validation_data),
-                               self._get_y(validation_data))
-
-        original_generator = generator
-
-        def fixed_generator():
-            while 1:
-                data = next(original_generator)
-                if type(data) is tuple:
-                    data, sample_weight = data
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y, sample_weight
-                else:
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y
-
-        generator = fixed_generator()
-        history = super(Graph, self).fit_generator(generator,
-                                                   samples_per_epoch,
-                                                   nb_epoch,
-                                                   verbose=verbose,
-                                                   callbacks=callbacks,
-                                                   validation_data=validation_data,
-                                                   nb_val_samples=nb_val_samples,
-                                                   class_weight=class_weight,
-                                                   max_q_size=max_q_size)
-        self.train_on_batch = self._train_on_batch
-        self.evaluate = self._evaluate
-        return history
-
-    def evaluate_generator(self, generator, val_samples,
-                           verbose=1, max_q_size=10, **kwargs):
-        '''Evaluates the model on a generator. The generator should
-        return the same kind of data with every yield as accepted
-        by `evaluate`.
-
-        If `show_accuracy`, it returns a tuple `(loss, accuracy)`,
-        otherwise it returns the loss value.
-
-        Arguments:
-            generator:
-                generator yielding dictionaries of the kind accepted
-                by `evaluate`, or tuples of such dictionaries and
-                associated dictionaries of sample weights.
-            val_samples:
-                total number of samples to generate from `generator`
-                to use in validation.
-
-            Other arguments are the same as for `fit`.
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if 'verbose' in kwargs:
-            kwargs.pop('verbose')
-            warnings.warn('The "verbose" argument is deprecated.')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-
-        self._test_on_batch = self.test_on_batch
-        self.test_on_batch = super(Graph, self).test_on_batch
-
-        original_generator = generator
-
-        def fixed_generator():
-            while 1:
-                data = next(original_generator)
-                if type(data) is tuple:
-                    data, sample_weight = data
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y, sample_weight
-                else:
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y
-
-        generator = fixed_generator()
-        history = super(Graph, self).evaluate_generator(generator,
-                                                        val_samples,
-                                                        max_q_size=max_q_size)
-        self.test_on_batch = self._test_on_batch
-        return history
-
-    # get_weights, set_weights: inherited
-    def get_config(self):
-        config = {'input_config': self._graph_input_config,
-                  'node_config': self._graph_node_config,
-                  'output_config': self._graph_output_config}
-        nodes = {}
-        for name, node in self._graph_nodes.items():
-            nodes[name] = {'class_name': node.__class__.__name__,
-                           'config': node.get_config()}
-            if name in self._graph_shared_nodes_names:
-                nodes[name]['shared'] = True
-        config['nodes'] = nodes
-        return copy.deepcopy(config)
-
-    @classmethod
-    def from_config(cls, config):
-        # TODO: test legacy support
-        from keras.utils.layer_utils import layer_from_config
-
-        def normalize_legacy_config(conf):
-            if 'class_name' not in conf:
-                class_name = conf['name']
-                name = conf.get('custom_name')
-                conf['name'] = name
-                new_config = {
-                    'class_name': class_name,
-                    'config': conf,
-                }
-                return new_config
-            return conf
-
-        graph = cls()
-        inputs = config.get('input_config')
-        for input in inputs:
-            graph.add_input(**input)
-
-        nodes = config.get('node_config')
-        for node in nodes:
-            layer_config = config['nodes'][node['name']]
-            layer_config = normalize_legacy_config(layer_config)
-            if 'layer' in node:
-                # for add_shared_node
-                node['layer'] = layer_from_config(node['layer'])
-            else:
-                layer = layer_from_config(layer_config)
-                node['layer'] = layer
-
-            node['create_output'] = False  # outputs will be added below
-            if layer_config.get('shared'):
-                graph.add_shared_node(**node)
-            else:
-                graph.add_node(**node)
-
-        outputs = config.get('output_config')
-        for output in outputs:
-            graph.add_output(**output)
-        return graph
-
-    def load_weights(self, fname):
-        if not self.built:
-            self.build()
-        super(Graph, self).load_weights(fname)

keras/metrics.py

@@ -1,5 +1,5 @@
-import numpy as np
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def binary_accuracy(y_true, y_pred):
@@ -8,7 +8,7 @@ def binary_accuracy(y_true, y_pred):
 
 def categorical_accuracy(y_true, y_pred):
     return K.mean(K.equal(K.argmax(y_true, axis=-1),
-                  K.argmax(y_pred, axis=-1)))
+                          K.argmax(y_pred, axis=-1)))
 
 
 def sparse_categorical_accuracy(y_true, y_pred):
@@ -16,6 +16,10 @@ def sparse_categorical_accuracy(y_true, y_pred):
                           K.cast(K.argmax(y_pred, axis=-1), K.floatx())))
 
 
+def top_k_categorical_accuracy(y_true, y_pred, k=5):
+    return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k))
+
+
 def mean_squared_error(y_true, y_pred):
     return K.mean(K.square(y_pred - y_true))
 
@@ -25,35 +29,31 @@ def mean_absolute_error(y_true, y_pred):
 
 
 def mean_absolute_percentage_error(y_true, y_pred):
-    diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), np.inf))
+    diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true),
+                                            K.epsilon(),
+                                            None))
     return 100. * K.mean(diff)
 
 
 def mean_squared_logarithmic_error(y_true, y_pred):
-    first_log = K.log(K.clip(y_pred, K.epsilon(), np.inf) + 1.)
-    second_log = K.log(K.clip(y_true, K.epsilon(), np.inf) + 1.)
+    first_log = K.log(K.clip(y_pred, K.epsilon(), None) + 1.)
+    second_log = K.log(K.clip(y_true, K.epsilon(), None) + 1.)
     return K.mean(K.square(first_log - second_log))
 
 
-def squared_hinge(y_true, y_pred):
-    return K.mean(K.square(K.maximum(1. - y_true * y_pred, 0.)))
-
-
 def hinge(y_true, y_pred):
     return K.mean(K.maximum(1. - y_true * y_pred, 0.))
 
 
+def squared_hinge(y_true, y_pred):
+    return K.mean(K.square(K.maximum(1. - y_true * y_pred, 0.)))
+
+
 def categorical_crossentropy(y_true, y_pred):
-    '''Expects a binary class matrix instead of a vector of scalar classes.
-    '''
     return K.mean(K.categorical_crossentropy(y_pred, y_true))
 
 
 def sparse_categorical_crossentropy(y_true, y_pred):
-    '''expects an array of integer classes.
-    Note: labels shape must have the same number of dimensions as output shape.
-    If you get a shape error, add a length-1 dimension to labels.
-    '''
     return K.mean(K.sparse_categorical_crossentropy(y_pred, y_true))
 
 
@@ -61,6 +61,12 @@ def binary_crossentropy(y_true, y_pred):
     return K.mean(K.binary_crossentropy(y_pred, y_true))
 
 
+def kullback_leibler_divergence(y_true, y_pred):
+    y_true = K.clip(y_true, K.epsilon(), 1)
+    y_pred = K.clip(y_pred, K.epsilon(), 1)
+    return K.mean(K.sum(y_true * K.log(y_true / y_pred), axis=-1))
+
+
 def poisson(y_true, y_pred):
     return K.mean(y_pred - y_true * K.log(y_pred + K.epsilon()))
 
@@ -72,8 +78,13 @@ def cosine_proximity(y_true, y_pred):
 
 
 def matthews_correlation(y_true, y_pred):
-    ''' Matthews correlation coefficient
-    '''
+    """Matthews correlation metric.
+
+    It is only computed as a batch-wise average, not globally.
+
+    Computes the Matthews correlation coefficient measure for quality
+    of binary classification problems.
+    """
     y_pred_pos = K.round(K.clip(y_pred, 0, 1))
     y_pred_neg = 1 - y_pred_pos
 
@@ -83,22 +94,91 @@ def matthews_correlation(y_true, y_pred):
     tp = K.sum(y_pos * y_pred_pos)
     tn = K.sum(y_neg * y_pred_neg)
 
-    fp = K.sum(1 - y_neg * y_pred_pos)
-    fn = K.sum(1 - y_pos * y_pred_neg)
-    
+    fp = K.sum(y_neg * y_pred_pos)
+    fn = K.sum(y_pos * y_pred_neg)
+
     numerator = (tp * tn - fp * fn)
     denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
 
     return numerator / (denominator + K.epsilon())
 
+
+def precision(y_true, y_pred):
+    """Precision metric.
+
+    Only computes a batch-wise average of precision.
+
+    Computes the precision, a metric for multi-label classification of
+    how many selected items are relevant.
+    """
+    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
+    precision = true_positives / (predicted_positives + K.epsilon())
+    return precision
+
+
+def recall(y_true, y_pred):
+    """Recall metric.
+
+    Only computes a batch-wise average of recall.
+
+    Computes the recall, a metric for multi-label classification of
+    how many relevant items are selected.
+    """
+    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
+    recall = true_positives / (possible_positives + K.epsilon())
+    return recall
+
+
+def fbeta_score(y_true, y_pred, beta=1):
+    """Computes the F score.
+
+    The F score is the weighted harmonic mean of precision and recall.
+    Here it is only computed as a batch-wise average, not globally.
+
+    This is useful for multi-label classification, where input samples can be
+    classified as sets of labels. By only using accuracy (precision) a model
+    would achieve a perfect score by simply assigning every class to every
+    input. In order to avoid this, a metric should penalize incorrect class
+    assignments as well (recall). The F-beta score (ranged from 0.0 to 1.0)
+    computes this, as a weighted mean of the proportion of correct class
+    assignments vs. the proportion of incorrect class assignments.
+
+    With beta = 1, this is equivalent to a F-measure. With beta < 1, assigning
+    correct classes becomes more important, and with beta > 1 the metric is
+    instead weighted towards penalizing incorrect class assignments.
+    """
+    if beta < 0:
+        raise ValueError('The lowest choosable beta is zero (only precision).')
+
+    # If there are no true positives, fix the F score at 0 like sklearn.
+    if K.sum(K.round(K.clip(y_true, 0, 1))) == 0:
+        return 0
+
+    p = precision(y_true, y_pred)
+    r = recall(y_true, y_pred)
+    bb = beta ** 2
+    fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())
+    return fbeta_score
+
+
+def fmeasure(y_true, y_pred):
+    """Computes the f-measure, the harmonic mean of precision and recall.
+
+    Here it is only computed as a batch-wise average, not globally.
+    """
+    return fbeta_score(y_true, y_pred, beta=1)
+
+
 # aliases
 mse = MSE = mean_squared_error
 mae = MAE = mean_absolute_error
 mape = MAPE = mean_absolute_percentage_error
 msle = MSLE = mean_squared_logarithmic_error
 cosine = cosine_proximity
+fscore = f1score = fmeasure
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier):
     return get_from_module(identifier, globals(), 'metric')

keras/models.py

@@ -6,11 +6,11 @@ import os
 import numpy as np
 
 from . import backend as K
+from . import optimizers
 from .utils.io_utils import ask_to_proceed_with_overwrite
 from .engine.training import Model
-from .engine.topology import get_source_inputs, Node
+from .engine.topology import get_source_inputs, Node, Layer, Merge
 from .optimizers import optimizer_from_config
-from .legacy.models import Graph
 
 
 def save_model(model, filepath, overwrite=True):
@@ -27,7 +27,7 @@ def save_model(model, filepath, overwrite=True):
             return obj.item()
 
         # misc functions (e.g. loss function)
-        if hasattr(obj, '__call__'):
+        if callable(obj):
             return obj.__name__
 
         # if obj is a python 'type'
@@ -56,48 +56,62 @@ def save_model(model, filepath, overwrite=True):
     model.save_weights_to_hdf5_group(model_weights_group)
 
     if hasattr(model, 'optimizer'):
-        f.attrs['training_config'] = json.dumps({
-            'optimizer_config': {
-                'class_name': model.optimizer.__class__.__name__,
-                'config': model.optimizer.get_config()
-            },
-            'loss': model.loss,
-            'metrics': model.metrics,
-            'sample_weight_mode': model.sample_weight_mode,
-            'loss_weights': model.loss_weights,
-        }, default=get_json_type).encode('utf8')
+        if isinstance(model.optimizer, optimizers.TFOptimizer):
+            warnings.warn(
+                'TensorFlow optimizers do not '
+                'make it possible to access '
+                'optimizer attributes or optimizer state '
+                'after instantiation. '
+                'As a result, we cannot save the optimizer '
+                'as part of the model save file.'
+                'You will have to compile your model again after loading it. '
+                'Prefer using a Keras optimizer instead '
+                '(see keras.io/optimizers).')
+        else:
+            f.attrs['training_config'] = json.dumps({
+                'optimizer_config': {
+                    'class_name': model.optimizer.__class__.__name__,
+                    'config': model.optimizer.get_config()
+                },
+                'loss': model.loss,
+                'metrics': model.metrics,
+                'sample_weight_mode': model.sample_weight_mode,
+                'loss_weights': model.loss_weights,
+            }, default=get_json_type).encode('utf8')
 
-        # save optimizer weights
-        symbolic_weights = getattr(model.optimizer, 'weights')
-        if symbolic_weights:
-            optimizer_weights_group = f.create_group('optimizer_weights')
-            weight_values = K.batch_get_value(symbolic_weights)
-            weight_names = []
-            for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
-                if hasattr(w, 'name') and w.name:
-                    name = str(w.name)
-                else:
-                    name = 'param_' + str(i)
-                weight_names.append(name.encode('utf8'))
-            optimizer_weights_group.attrs['weight_names'] = weight_names
-            for name, val in zip(weight_names, weight_values):
-                param_dset = optimizer_weights_group.create_dataset(
-                    name,
-                    val.shape,
-                    dtype=val.dtype)
-                if not val.shape:
-                    # scalar
-                    param_dset[()] = val
-                else:
-                    param_dset[:] = val
+            # save optimizer weights
+            symbolic_weights = getattr(model.optimizer, 'weights')
+            if symbolic_weights:
+                optimizer_weights_group = f.create_group('optimizer_weights')
+                weight_values = K.batch_get_value(symbolic_weights)
+                weight_names = []
+                for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
+                    if hasattr(w, 'name') and w.name:
+                        name = str(w.name)
+                    else:
+                        name = 'param_' + str(i)
+                    weight_names.append(name.encode('utf8'))
+                optimizer_weights_group.attrs['weight_names'] = weight_names
+                for name, val in zip(weight_names, weight_values):
+                    param_dset = optimizer_weights_group.create_dataset(
+                        name,
+                        val.shape,
+                        dtype=val.dtype)
+                    if not val.shape:
+                        # scalar
+                        param_dset[()] = val
+                    else:
+                        param_dset[:] = val
     f.flush()
     f.close()
 
 
-def load_model(filepath, custom_objects={}):
+def load_model(filepath, custom_objects=None):
+    if not custom_objects:
+        custom_objects = {}
 
     def deserialize(obj):
-        if type(obj) is list:
+        if isinstance(obj, list):
             deserialized = []
             for value in obj:
                 if value in custom_objects:
@@ -105,7 +119,7 @@ def load_model(filepath, custom_objects={}):
                 else:
                     deserialized.append(value)
             return deserialized
-        if type(obj) is dict:
+        if isinstance(obj, dict):
             deserialized = {}
             for key, value in obj.items():
                 if value in custom_objects:
@@ -139,7 +153,8 @@ def load_model(filepath, custom_objects={}):
         return model
     training_config = json.loads(training_config.decode('utf-8'))
     optimizer_config = training_config['optimizer_config']
-    optimizer = optimizer_from_config(optimizer_config)
+    optimizer = optimizer_from_config(optimizer_config,
+                                      custom_objects=custom_objects)
 
     # recover loss functions and metrics
     loss = deserialize(training_config['loss'])
@@ -157,7 +172,7 @@ def load_model(filepath, custom_objects={}):
     # set optimizer weights
     if 'optimizer_weights' in f:
         # build train function (to get weight updates)
-        if model.__class__.__name__ == 'Sequential':
+        if isinstance(model, Sequential):
             model.model._make_train_function()
         else:
             model._make_train_function()
@@ -169,28 +184,29 @@ def load_model(filepath, custom_objects={}):
     return model
 
 
-def model_from_config(config, custom_objects={}):
+def model_from_config(config, custom_objects=None):
     from keras.utils.layer_utils import layer_from_config
     if isinstance(config, list):
-        raise Exception('`model_fom_config` expects a dictionary, not a list. '
-                        'Maybe you meant to use `Sequential.from_config(config)`?')
+        raise TypeError('`model_fom_config` expects a dictionary, not a list. '
+                        'Maybe you meant to use '
+                        '`Sequential.from_config(config)`?')
     return layer_from_config(config, custom_objects=custom_objects)
 
 
-def model_from_yaml(yaml_string, custom_objects={}):
-    '''Parses a yaml model configuration file
+def model_from_yaml(yaml_string, custom_objects=None):
+    """Parses a yaml model configuration file
     and returns a model instance.
-    '''
+    """
     import yaml
     from keras.utils.layer_utils import layer_from_config
     config = yaml.load(yaml_string)
     return layer_from_config(config, custom_objects=custom_objects)
 
 
-def model_from_json(json_string, custom_objects={}):
-    '''Parses a JSON model configuration file
+def model_from_json(json_string, custom_objects=None):
+    """Parses a JSON model configuration file
     and returns a model instance.
-    '''
+    """
     import json
     from keras.utils.layer_utils import layer_from_config
     config = json.loads(json_string)
@@ -198,7 +214,7 @@ def model_from_json(json_string, custom_objects={}):
 
 
 class Sequential(Model):
-    '''Linear stack of layers.
+    """Linear stack of layers.
 
     # Arguments
         layers: list of layers to add to the model.
@@ -232,13 +248,14 @@ class Sequential(Model):
             model.add(Dense(32, batch_input_shape=(None, 500)))
             model.add(Dense(32))
         ```
-    '''
-    def __init__(self, layers=[], name=None):
+    """
+
+    def __init__(self, layers=None, name=None):
         self.layers = []  # stack of layers
         self.model = None  # internal Model instance
         self.inputs = []  # tensors
         self.outputs = []  # tensors (length 1)
-        self.trainable = True
+        self._trainable = True
 
         # model attributes
         self.inbound_nodes = []
@@ -251,23 +268,29 @@ class Sequential(Model):
             name = prefix + str(K.get_uid(prefix))
         self.name = name
 
-        for layer in layers:
-            self.add(layer)
+        if layers:
+            for layer in layers:
+                self.add(layer)
 
     def add(self, layer):
-        '''Adds a layer instance on top of the layer stack.
+        """Adds a layer instance on top of the layer stack.
 
         # Arguments
             layer: layer instance.
-        '''
+        """
+        if not isinstance(layer, Layer):
+            raise TypeError('The added layer must be '
+                            'an instance of class Layer. '
+                            'Found: ' + str(layer))
         if not self.outputs:
             # first layer in model: check that it is an input layer
             if len(layer.inbound_nodes) == 0:
                 # create an input layer
                 if not hasattr(layer, 'batch_input_shape'):
-                    raise Exception('The first layer in a Sequential model must '
-                                    'get an `input_shape` or '
-                                    '`batch_input_shape` argument.')
+                    raise ValueError('The first layer in a '
+                                     'Sequential model must '
+                                     'get an `input_shape` or '
+                                     '`batch_input_shape` argument.')
                 batch_input_shape = layer.batch_input_shape
                 if hasattr(layer, 'input_dtype'):
                     input_dtype = layer.input_dtype
@@ -276,17 +299,18 @@ class Sequential(Model):
                 layer.create_input_layer(batch_input_shape, input_dtype)
 
             if len(layer.inbound_nodes) != 1:
-                raise Exception('A layer added to a Sequential model must '
-                                'not already be connected somewhere else. '
-                                'Model received layer ' + layer.name +
-                                ' which has ' + str(len(layer.inbound_nodes)) +
-                                ' pre-existing inbound connections.')
+                raise ValueError('A layer added to a Sequential model must '
+                                 'not already be connected somewhere else. '
+                                 'Model received layer ' + layer.name +
+                                 ' which has ' +
+                                 str(len(layer.inbound_nodes)) +
+                                 ' pre-existing inbound connections.')
 
             if len(layer.inbound_nodes[0].output_tensors) != 1:
-                raise Exception('All layers in a Sequential model '
-                                'should have a single output tensor. '
-                                'For multi-output layers, '
-                                'use the functional API.')
+                raise ValueError('All layers in a Sequential model '
+                                 'should have a single output tensor. '
+                                 'For multi-output layers, '
+                                 'use the functional API.')
 
             self.outputs = [layer.inbound_nodes[0].output_tensors[0]]
             self.inputs = get_source_inputs(self.outputs[0])
@@ -306,8 +330,8 @@ class Sequential(Model):
                  output_shapes=[self.outputs[0]._keras_shape])
         else:
             output_tensor = layer(self.outputs[0])
-            if type(output_tensor) is list:
-                raise Exception('All layers in a Sequential model '
+            if isinstance(output_tensor, list):
+                raise TypeError('All layers in a Sequential model '
                                 'should have a single output tensor. '
                                 'For multi-output layers, '
                                 'use the functional API.')
@@ -321,10 +345,10 @@ class Sequential(Model):
         self._flattened_layers = None
 
     def pop(self):
-        '''Removes the last layer in the model.
-        '''
+        """Removes the last layer in the model.
+        """
         if not self.layers:
-            raise Exception('There are no layers in the model.')
+            raise TypeError('There are no layers in the model.')
 
         self.layers.pop()
         if not self.layers:
@@ -341,7 +365,7 @@ class Sequential(Model):
         self._flattened_layers = None
 
     def get_layer(self, name=None, index=None):
-        '''Returns a layer based on either its name (unique)
+        """Returns a layer based on either its name (unique)
         or its index in the graph. Indices are based on
         order of horizontal graph traversal (bottom-up).
 
@@ -351,7 +375,7 @@ class Sequential(Model):
 
         # Returns
             A layer instance.
-        '''
+        """
         if not self.built:
             self.build()
         return self.model.get_layer(name, index)
@@ -363,10 +387,12 @@ class Sequential(Model):
 
     def build(self, input_shape=None):
         if not self.inputs or not self.outputs:
-            raise Exception('Sequential model cannot be built: model is empty.'
+            raise TypeError('Sequential model cannot be built: model is empty.'
                             ' Add some layers first.')
         # actually create the model
-        self.model = Model(self.inputs, self.outputs[0], name=self.name + '_model')
+        self.model = Model(self.inputs, self.outputs[0],
+                           name=self.name + '_model')
+        self.model.trainable = self.trainable
 
         # mirror model attributes
         self.supports_masking = self.model.supports_masking
@@ -400,26 +426,27 @@ class Sequential(Model):
         if self._flattened_layers is not None:
             return self._flattened_layers
         layers = []
-        if self.layers[0].__class__.__name__ == 'Merge':
-            merge = self.layers[0]
-            for layer in merge.layers:
-                if hasattr(layer, 'flattened_layers'):
-                    for sublayer in layer.flattened_layers:
-                        if sublayer not in layers:
-                            layers.append(sublayer)
-                elif hasattr(layer, 'layers'):
-                    for sublayer in layer.layers:
-                        if sublayer not in layers:
-                            layers.append(sublayer)
-                else:
-                    if layer not in layers:
-                        layers.append(layer)
-        else:
-            if self.layers[0] not in layers:
-                layers.append(self.layers[0])
-        for layer in self.layers[1:]:
-            if layer not in layers:
-                layers.append(layer)
+        if self.layers:
+            if isinstance(self.layers[0], Merge):
+                merge = self.layers[0]
+                for layer in merge.layers:
+                    if hasattr(layer, 'flattened_layers'):
+                        for sublayer in layer.flattened_layers:
+                            if sublayer not in layers:
+                                layers.append(sublayer)
+                    elif hasattr(layer, 'layers'):
+                        for sublayer in layer.layers:
+                            if sublayer not in layers:
+                                layers.append(sublayer)
+                    else:
+                        if layer not in layers:
+                            layers.append(layer)
+            else:
+                if self.layers[0] not in layers:
+                    layers.append(self.layers[0])
+            for layer in self.layers[1:]:
+                if layer not in layers:
+                    layers.append(layer)
         self._flattened_layers = layers
         return layers
 
@@ -437,6 +464,16 @@ class Sequential(Model):
                              list(layer_dict.items()))
         return all_attrs
 
+    @property
+    def trainable(self):
+        return self._trainable
+
+    @trainable.setter
+    def trainable(self, value):
+        if self.model:
+            self.model.trainable = value
+        self._trainable = value
+
     @property
     def trainable_weights(self):
         if not self.trainable:
@@ -455,13 +492,22 @@ class Sequential(Model):
 
     @property
     def updates(self):
-        # support for legacy behavior
-        return self._gather_list_attr('updates')
+        return self.model.updates
 
     @property
     def state_updates(self):
         # support for legacy behavior
-        return self._gather_list_attr('state_updates')
+        return self.model.state_updates
+
+    def get_updates_for(self, inputs):
+        return self.model.get_updates_for(inputs)
+
+    @property
+    def losses(self):
+        return self.model.losses
+
+    def get_losses_for(self, inputs):
+        return self.model.get_losses_for(inputs)
 
     @property
     def regularizers(self):
@@ -474,9 +520,9 @@ class Sequential(Model):
         return self._gather_dict_attr('constraints')
 
     def get_weights(self):
-        '''Returns the weights of the model,
+        """Returns the weights of the model,
         as a flat list of Numpy arrays.
-        '''
+        """
         # support for legacy behavior
         weights = []
         for layer in self.flattened_layers:
@@ -484,11 +530,11 @@ class Sequential(Model):
         return weights
 
     def set_weights(self, weights):
-        '''Sets the weights of the model.
+        """Sets the weights of the model.
         The `weights` argument should be a list
         of Numpy arrays with shapes and types matching
         the output of `model.get_weights()`.
-        '''
+        """
         # support for legacy behavior
         for layer in self.flattened_layers:
             nb_param = len(layer.weights)
@@ -504,10 +550,10 @@ class Sequential(Model):
         return self.model.training_data
 
     def compile(self, optimizer, loss,
-                metrics=[],
+                metrics=None,
                 sample_weight_mode=None,
                 **kwargs):
-        '''Configures the learning process.
+        """Configures the learning process.
 
         # Arguments
             optimizer: str (name of optimizer) or optimizer object.
@@ -517,6 +563,7 @@ class Sequential(Model):
             metrics: list of metrics to be evaluated by the model
                 during training and testing.
                 Typically you will use `metrics=['accuracy']`.
+                See [metrics](/metrics).
             sample_weight_mode: if you need to do timestep-wise
                 sample weighting (2D weights), set this to "temporal".
                 "None" defaults to sample-wise weights (1D).
@@ -532,7 +579,7 @@ class Sequential(Model):
                               loss='categorical_crossentropy',
                               metrics=['accuracy'])
             ```
-        '''
+        """
         # create the underlying model
         self.build()
         # legacy kwarg support
@@ -553,10 +600,10 @@ class Sequential(Model):
         self.metrics_names = self.model.metrics_names
         self.sample_weight_mode = self.model.sample_weight_mode
 
-    def fit(self, x, y, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
+    def fit(self, x, y, batch_size=32, nb_epoch=10, verbose=1, callbacks=None,
             validation_split=0., validation_data=None, shuffle=True,
-            class_weight=None, sample_weight=None, **kwargs):
-        '''Trains the model for a fixed number of epochs.
+            class_weight=None, sample_weight=None, initial_epoch=0, **kwargs):
+        """Trains the model for a fixed number of epochs.
 
         # Arguments
             x: input data, as a Numpy array or list of Numpy arrays
@@ -571,7 +618,8 @@ class Sequential(Model):
                 See [callbacks](/callbacks).
             validation_split: float (0. < x < 1).
                 Fraction of the data to use as held-out validation data.
-            validation_data: tuple (X, y) to be used as held-out
+            validation_data: tuple (x_val, y_val) or tuple
+                (x_val, y_val, val_sample_weights) to be used as held-out
                 validation data. Will override validation_split.
             shuffle: boolean or str (for 'batch').
                 Whether to shuffle the samples at each epoch.
@@ -589,15 +637,18 @@ class Sequential(Model):
                 to apply a different weight to every timestep of every sample.
                 In this case you should make sure to specify
                 sample_weight_mode="temporal" in compile().
+            initial_epoch: epoch at which to start training
+                (useful for resuming a previous training run)
 
         # Returns
             A `History` object. Its `History.history` attribute is
             a record of training loss values and metrics values
             at successive epochs, as well as validation loss values
             and validation metrics values (if applicable).
-        '''
+        """
         if self.model is None:
-            raise Exception('The model needs to be compiled before being used.')
+            raise RuntimeError('The model needs to be compiled '
+                               'before being used.')
         if 'show_accuracy' in kwargs:
             kwargs.pop('show_accuracy')
             warnings.warn('The "show_accuracy" argument is deprecated, '
@@ -606,7 +657,7 @@ class Sequential(Model):
                           '`model.compile(optimizer, loss, '
                           'metrics=["accuracy"])`')
         if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
+            raise TypeError('Received unknown keyword arguments: ' +
                             str(kwargs))
         return self.model.fit(x, y,
                               batch_size=batch_size,
@@ -617,11 +668,12 @@ class Sequential(Model):
                               validation_data=validation_data,
                               shuffle=shuffle,
                               class_weight=class_weight,
-                              sample_weight=sample_weight)
+                              sample_weight=sample_weight,
+                              initial_epoch=initial_epoch)
 
     def evaluate(self, x, y, batch_size=32, verbose=1,
                  sample_weight=None, **kwargs):
-        '''Computes the loss on some input data, batch by batch.
+        """Computes the loss on some input data, batch by batch.
 
         # Arguments
             x: input data, as a Numpy array or list of Numpy arrays
@@ -636,9 +688,10 @@ class Sequential(Model):
             or list of scalars (if the model computes other metrics).
             The attribute `model.metrics_names` will give you
             the display labels for the scalar outputs.
-        '''
+        """
         if self.model is None:
-            raise Exception('The model needs to be compiled before being used.')
+            raise RuntimeError('The model needs to be compiled '
+                               'before being used.')
         if 'show_accuracy' in kwargs:
             kwargs.pop('show_accuracy')
             warnings.warn('The "show_accuracy" argument is deprecated, '
@@ -647,7 +700,7 @@ class Sequential(Model):
                           '`model.compile(optimizer, loss, '
                           'metrics=["accuracy"])`')
         if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
+            raise TypeError('Received unknown keyword arguments: ' +
                             str(kwargs))
         return self.model.evaluate(x, y,
                                    batch_size=batch_size,
@@ -655,7 +708,7 @@ class Sequential(Model):
                                    sample_weight=sample_weight)
 
     def predict(self, x, batch_size=32, verbose=0):
-        '''Generates output predictions for the input samples,
+        """Generates output predictions for the input samples,
         processing the samples in a batched way.
 
         # Arguments
@@ -665,21 +718,21 @@ class Sequential(Model):
 
         # Returns
             A Numpy array of predictions.
-        '''
+        """
         if self.model is None:
             self.build()
         return self.model.predict(x, batch_size=batch_size, verbose=verbose)
 
     def predict_on_batch(self, x):
-        '''Returns predictions for a single batch of samples.
-        '''
+        """Returns predictions for a single batch of samples.
+        """
         if self.model is None:
             self.build()
         return self.model.predict_on_batch(x)
 
     def train_on_batch(self, x, y, class_weight=None,
                        sample_weight=None, **kwargs):
-        '''Single gradient update over one batch of samples.
+        """Single gradient update over one batch of samples.
 
         # Arguments
             x: input data, as a Numpy array or list of Numpy arrays
@@ -694,9 +747,10 @@ class Sequential(Model):
             or list of scalars (if the model computes other metrics).
             The attribute `model.metrics_names` will give you
             the display labels for the scalar outputs.
-        '''
+        """
         if self.model is None:
-            raise Exception('The model needs to be compiled before being used.')
+            raise RuntimeError('The model needs to be compiled '
+                               'before being used.')
         if 'accuracy' in kwargs:
             kwargs.pop('accuracy')
             warnings.warn('The "accuracy" argument is deprecated, '
@@ -705,7 +759,7 @@ class Sequential(Model):
                           '`model.compile(optimizer, loss, '
                           'metrics=["accuracy"])`')
         if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
+            raise TypeError('Received unknown keyword arguments: ' +
                             str(kwargs))
         return self.model.train_on_batch(x, y,
                                          sample_weight=sample_weight,
@@ -713,7 +767,7 @@ class Sequential(Model):
 
     def test_on_batch(self, x, y,
                       sample_weight=None, **kwargs):
-        '''Evaluates the model over a single batch of samples.
+        """Evaluates the model over a single batch of samples.
 
         # Arguments
             x: input data, as a Numpy array or list of Numpy arrays
@@ -726,9 +780,10 @@ class Sequential(Model):
             or list of scalars (if the model computes other metrics).
             The attribute `model.metrics_names` will give you
             the display labels for the scalar outputs.
-        '''
+        """
         if self.model is None:
-            raise Exception('The model needs to be compiled before being used.')
+            raise RuntimeError('The model needs to be compiled '
+                               'before being used.')
         if 'accuracy' in kwargs:
             kwargs.pop('accuracy')
             warnings.warn('The "accuracy" argument is deprecated, '
@@ -737,13 +792,13 @@ class Sequential(Model):
                           '`model.compile(optimizer, loss, '
                           'metrics=["accuracy"])`')
         if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
+            raise TypeError('Received unknown keyword arguments: ' +
                             str(kwargs))
         return self.model.test_on_batch(x, y,
                                         sample_weight=sample_weight)
 
     def predict_proba(self, x, batch_size=32, verbose=1):
-        '''Generates class probability predictions for the input samples
+        """Generates class probability predictions for the input samples
         batch by batch.
 
         # Arguments
@@ -754,7 +809,7 @@ class Sequential(Model):
 
         # Returns
             A Numpy array of probability predictions.
-        '''
+        """
         preds = self.predict(x, batch_size, verbose)
         if preds.min() < 0. or preds.max() > 1.:
             warnings.warn('Network returning invalid probability values. '
@@ -764,7 +819,7 @@ class Sequential(Model):
         return preds
 
     def predict_classes(self, x, batch_size=32, verbose=1):
-        '''Generate class predictions for the input samples
+        """Generate class predictions for the input samples
         batch by batch.
 
         # Arguments
@@ -775,7 +830,7 @@ class Sequential(Model):
 
         # Returns
             A numpy array of class predictions.
-        '''
+        """
         proba = self.predict(x, batch_size=batch_size, verbose=verbose)
         if proba.shape[-1] > 1:
             return proba.argmax(axis=-1)
@@ -783,10 +838,11 @@ class Sequential(Model):
             return (proba > 0.5).astype('int32')
 
     def fit_generator(self, generator, samples_per_epoch, nb_epoch,
-                      verbose=1, callbacks=[],
+                      verbose=1, callbacks=None,
                       validation_data=None, nb_val_samples=None,
-                      class_weight=None, max_q_size=10, nb_worker=1, pickle_safe=False, **kwargs):
-        '''Fits the model on data generated batch-by-batch by
+                      class_weight=None, max_q_size=10, nb_worker=1,
+                      pickle_safe=False, initial_epoch=0, **kwargs):
+        """Fits the model on data generated batch-by-batch by
         a Python generator.
         The generator is run in parallel to the model, for efficiency.
         For instance, this allows you to do real-time data augmentation
@@ -821,6 +877,8 @@ class Sequential(Model):
                 this implementation relies on multiprocessing, you should not pass
                 non picklable arguments to the generator as they can't be passed
                 easily to children processes.
+            initial_epoch: epoch at which to start training
+                (useful for resuming a previous training run)
 
         # Returns
             A `History` object.
@@ -841,11 +899,13 @@ class Sequential(Model):
             model.fit_generator(generate_arrays_from_file('/my_file.txt'),
                                 samples_per_epoch=10000, nb_epoch=10)
         ```
-        '''
+        """
         if self.model is None:
-            raise Exception('The model needs to be compiled before being used.')
+            raise RuntimeError('The model needs to be compiled '
+                               'before being used.')
         if nb_worker > 1 and not pickle_safe:
-            warnings.warn('The "nb_worker" argument is deprecated when pickle_safe is False')
+            warnings.warn('The "nb_worker" argument is deprecated '
+                          'when pickle_safe is False')
             nb_worker = 1  # For backward compatibility
         if 'show_accuracy' in kwargs:
             kwargs.pop('show_accuracy')
@@ -859,7 +919,7 @@ class Sequential(Model):
             warnings.warn('The "nb_val_worker" argument is deprecated, '
                           'please remove it from your code.')
         if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
+            raise TypeError('Received unknown keyword arguments: ' +
                             str(kwargs))
         return self.model.fit_generator(generator,
                                         samples_per_epoch,
@@ -871,10 +931,13 @@ class Sequential(Model):
                                         class_weight=class_weight,
                                         max_q_size=max_q_size,
                                         nb_worker=nb_worker,
-                                        pickle_safe=pickle_safe)
+                                        pickle_safe=pickle_safe,
+                                        initial_epoch=initial_epoch)
 
-    def evaluate_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False, **kwargs):
-        '''Evaluates the model on a data generator. The generator should
+    def evaluate_generator(self, generator, val_samples,
+                           max_q_size=10, nb_worker=1,
+                           pickle_safe=False, **kwargs):
+        """Evaluates the model on a data generator. The generator should
         return the same kind of data as accepted by `test_on_batch`.
 
         # Arguments
@@ -890,11 +953,13 @@ class Sequential(Model):
                 this implementation relies on multiprocessing, you should not pass non
                 non picklable arguments to the generator as they can't be passed
                 easily to children processes.
-        '''
+        """
         if self.model is None:
-            raise Exception('The model needs to be compiled before being used.')
+            raise RuntimeError('The model needs to be compiled '
+                               'before being used.')
         if nb_worker > 1 and not pickle_safe:
-            warnings.warn('The "nb_worker" argument is deprecated when pickle_safe is False')
+            warnings.warn('The "nb_worker" argument is deprecated '
+                          'when pickle_safe is False')
             nb_worker = 1  # For backward compatibility
         if 'show_accuracy' in kwargs:
             kwargs.pop('show_accuracy')
@@ -907,7 +972,7 @@ class Sequential(Model):
             kwargs.pop('verbose')
             warnings.warn('The "verbose" argument is deprecated.')
         if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
+            raise TypeError('Received unknown keyword arguments: ' +
                             str(kwargs))
         return self.model.evaluate_generator(generator,
                                              val_samples,
@@ -915,8 +980,9 @@ class Sequential(Model):
                                              nb_worker=nb_worker,
                                              pickle_safe=pickle_safe)
 
-    def predict_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False):
-        '''Generates predictions for the input samples from a data generator.
+    def predict_generator(self, generator, val_samples,
+                          max_q_size=10, nb_worker=1, pickle_safe=False):
+        """Generates predictions for the input samples from a data generator.
         The generator should return the same kind of data as accepted by
         `predict_on_batch`.
 
@@ -933,11 +999,12 @@ class Sequential(Model):
 
         # Returns
             A Numpy array of predictions.
-        '''
+        """
         if self.model is None:
             self.build()
         if nb_worker > 1 and not pickle_safe:
-            warnings.warn('The "nb_worker" argument is deprecated when pickle_safe is False')
+            warnings.warn('The "nb_worker" argument is deprecated '
+                          'when pickle_safe is False')
             nb_worker = 1  # For backward compatibility
         return self.model.predict_generator(generator, val_samples,
                                             max_q_size=max_q_size,
@@ -945,11 +1012,11 @@ class Sequential(Model):
                                             pickle_safe=pickle_safe)
 
     def get_config(self):
-        '''Returns the model configuration
+        """Returns the model configuration
         as a Python list.
-        '''
+        """
         config = []
-        if self.layers[0].__class__.__name__ == 'Merge':
+        if isinstance(self.layers[0], Merge):
             assert hasattr(self.layers[0], 'layers')
             layers = []
             for layer in self.layers[0].layers:
@@ -969,11 +1036,10 @@ class Sequential(Model):
 
     @classmethod
     def from_config(cls, config, layer_cache=None):
-        '''Supports legacy formats
-        '''
+        """Supports legacy formats
+        """
         from keras.utils.layer_utils import layer_from_config
         from keras.layers import Merge
-        assert type(config) is list
 
         if not layer_cache:
             layer_cache = {}

keras/objectives.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
-import numpy as np
+
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def mean_squared_error(y_true, y_pred):
@@ -12,13 +13,15 @@ def mean_absolute_error(y_true, y_pred):
 
 
 def mean_absolute_percentage_error(y_true, y_pred):
-    diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), np.inf))
+    diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true),
+                                            K.epsilon(),
+                                            None))
     return 100. * K.mean(diff, axis=-1)
 
 
 def mean_squared_logarithmic_error(y_true, y_pred):
-    first_log = K.log(K.clip(y_pred, K.epsilon(), np.inf) + 1.)
-    second_log = K.log(K.clip(y_true, K.epsilon(), np.inf) + 1.)
+    first_log = K.log(K.clip(y_pred, K.epsilon(), None) + 1.)
+    second_log = K.log(K.clip(y_true, K.epsilon(), None) + 1.)
     return K.mean(K.square(first_log - second_log), axis=-1)
 
 
@@ -31,16 +34,10 @@ def hinge(y_true, y_pred):
 
 
 def categorical_crossentropy(y_true, y_pred):
-    '''Expects a binary class matrix instead of a vector of scalar classes.
-    '''
     return K.categorical_crossentropy(y_pred, y_true)
 
 
 def sparse_categorical_crossentropy(y_true, y_pred):
-    '''expects an array of integer classes.
-    Note: labels shape must have the same number of dimensions as output shape.
-    If you get a shape error, add a length-1 dimension to labels.
-    '''
     return K.sparse_categorical_crossentropy(y_pred, y_true)
 
 
@@ -64,7 +61,8 @@ def cosine_proximity(y_true, y_pred):
     return -K.mean(y_true * y_pred, axis=-1)
 
 
-# aliases
+# Aliases.
+
 mse = MSE = mean_squared_error
 mae = MAE = mean_absolute_error
 mape = MAPE = mean_absolute_percentage_error
@@ -72,6 +70,6 @@ msle = MSLE = mean_squared_logarithmic_error
 kld = KLD = kullback_leibler_divergence
 cosine = cosine_proximity
 
-from .utils.generic_utils import get_from_module
+
 def get(identifier):
     return get_from_module(identifier, globals(), 'objective')

keras/optimizers.py

@@ -1,8 +1,13 @@
 from __future__ import absolute_import
-from . import backend as K
-from .utils.generic_utils import get_from_module
+
 from six.moves import zip
 
+from . import backend as K
+from .utils.generic_utils import get_from_module, get_custom_objects
+
+if K.backend() == 'tensorflow':
+    import tensorflow as tf
+
 
 def clip_norm(g, c, n):
     if c > 0:
@@ -10,7 +15,20 @@ def clip_norm(g, c, n):
     return g
 
 
-def optimizer_from_config(config, custom_objects={}):
+def optimizer_from_config(config, custom_objects=None):
+    """Instantiate an optimizer given a config dictionary.
+
+    # Arguments
+        config: Config dictionary
+            (e.g. output of `optimizer.get_config()`).
+        custom_objects: Optional dictionary of custom optimizer classes.
+
+    # Returns
+        An optimizer instance.
+
+    # Raises
+        ValueError: in case of invalid optimizer config.
+    """
     all_classes = {
         'sgd': SGD,
         'rmsprop': RMSprop,
@@ -19,10 +37,13 @@ def optimizer_from_config(config, custom_objects={}):
         'adam': Adam,
         'adamax': Adamax,
         'nadam': Nadam,
+        'tfoptimizer': TFOptimizer,
     }
     class_name = config['class_name']
-    if class_name in custom_objects:
+    if custom_objects and class_name in custom_objects:
         cls = custom_objects[class_name]
+    elif class_name in get_custom_objects():
+        cls = get_custom_objects()[class_name]
     else:
         if class_name.lower() not in all_classes:
             raise ValueError('Optimizer class not found:', class_name)
@@ -31,7 +52,7 @@ def optimizer_from_config(config, custom_objects={}):
 
 
 class Optimizer(object):
-    '''Abstract optimizer base class.
+    """Abstract optimizer base class.
 
     Note: this is the parent class of all optimizers, not an actual optimizer
     that can be used for training models.
@@ -42,25 +63,18 @@ class Optimizer(object):
             when their L2 norm exceeds this value.
         clipvalue: float >= 0. Gradients will be clipped
             when their absolute value exceeds this value.
-    '''
+    """
+
     def __init__(self, **kwargs):
         allowed_kwargs = {'clipnorm', 'clipvalue'}
         for k in kwargs:
             if k not in allowed_kwargs:
-                raise Exception('Unexpected keyword argument '
+                raise TypeError('Unexpected keyword argument '
                                 'passed to optimizer: ' + str(k))
         self.__dict__.update(kwargs)
         self.updates = []
         self.weights = []
 
-    def get_state(self):
-        return [K.get_value(u[0]) for u in self.updates]
-
-    def set_state(self, value_list):
-        assert len(self.updates) == len(value_list)
-        for u, v in zip(self.updates, value_list):
-            K.set_value(u[0], v)
-
     def get_updates(self, params, constraints, loss):
         raise NotImplementedError
 
@@ -74,7 +88,7 @@ class Optimizer(object):
         return grads
 
     def set_weights(self, weights):
-        '''Sets the weights of the optimizer, from Numpy arrays.
+        """Sets the weights of the optimizer, from Numpy arrays.
 
         Should only be called after computing the gradients
         (otherwise the optimizer has no weights).
@@ -85,23 +99,28 @@ class Optimizer(object):
                 number of the dimensions of the weights
                 of the optimizer (i.e. it should match the
                 output of `get_weights`).
-        '''
+
+        # Raises
+            ValueError: in case of incompatible weight shapes.
+        """
         params = self.weights
         weight_value_tuples = []
         param_values = K.batch_get_value(params)
         for pv, p, w in zip(param_values, params, weights):
             if pv.shape != w.shape:
-                raise Exception('Optimizer weight shape ' +
-                                str(pv.shape) +
-                                ' not compatible with '
-                                'provided weight shape ' + str(w.shape))
+                raise ValueError('Optimizer weight shape ' +
+                                 str(pv.shape) +
+                                 ' not compatible with '
+                                 'provided weight shape ' + str(w.shape))
             weight_value_tuples.append((p, w))
         K.batch_set_value(weight_value_tuples)
 
     def get_weights(self):
-        '''Returns the current weights of the optimizer,
-        as a list of numpy arrays.
-        '''
+        """Returns the current value of the weights of the optimizer.
+
+        # Returns
+            A list of numpy arrays.
+        """
         return K.batch_get_value(self.weights)
 
     def get_config(self):
@@ -118,7 +137,9 @@ class Optimizer(object):
 
 
 class SGD(Optimizer):
-    '''Stochastic gradient descent, with support for momentum,
+    """Stochastic gradient descent optimizer.
+
+    Includes support for momentum,
     learning rate decay, and Nesterov momentum.
 
     # Arguments
@@ -126,23 +147,24 @@ class SGD(Optimizer):
         momentum: float >= 0. Parameter updates momentum.
         decay: float >= 0. Learning rate decay over each update.
         nesterov: boolean. Whether to apply Nesterov momentum.
-    '''
+    """
+
     def __init__(self, lr=0.01, momentum=0., decay=0.,
                  nesterov=False, **kwargs):
         super(SGD, self).__init__(**kwargs)
-        self.__dict__.update(locals())
         self.iterations = K.variable(0.)
         self.lr = K.variable(lr)
         self.momentum = K.variable(momentum)
         self.decay = K.variable(decay)
-        self.inital_decay = decay
+        self.initial_decay = decay
+        self.nesterov = nesterov
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
         self.updates = []
 
         lr = self.lr
-        if self.inital_decay > 0:
+        if self.initial_decay > 0:
             lr *= (1. / (1. + self.decay * self.iterations))
             self.updates .append(K.update_add(self.iterations, 1))
 
@@ -177,7 +199,7 @@ class SGD(Optimizer):
 
 
 class RMSprop(Optimizer):
-    '''RMSProp optimizer.
+    """RMSProp optimizer.
 
     It is recommended to leave the parameters of this optimizer
     at their default values
@@ -191,15 +213,19 @@ class RMSprop(Optimizer):
         rho: float >= 0.
         epsilon: float >= 0. Fuzz factor.
         decay: float >= 0. Learning rate decay over each update.
-    '''
+
+    # References
+        - [rmsprop: Divide the gradient by a running average of its recent magnitude](http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf)
+    """
+
     def __init__(self, lr=0.001, rho=0.9, epsilon=1e-8, decay=0.,
                  **kwargs):
         super(RMSprop, self).__init__(**kwargs)
-        self.__dict__.update(locals())
         self.lr = K.variable(lr)
         self.rho = K.variable(rho)
+        self.epsilon = epsilon
         self.decay = K.variable(decay)
-        self.inital_decay = decay
+        self.initial_decay = decay
         self.iterations = K.variable(0.)
 
     def get_updates(self, params, constraints, loss):
@@ -210,7 +236,7 @@ class RMSprop(Optimizer):
         self.updates = []
 
         lr = self.lr
-        if self.inital_decay > 0:
+        if self.initial_decay > 0:
             lr *= (1. / (1. + self.decay * self.iterations))
             self.updates.append(K.update_add(self.iterations, 1))
 
@@ -230,13 +256,14 @@ class RMSprop(Optimizer):
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
                   'rho': float(K.get_value(self.rho)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(RMSprop, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
 
 class Adagrad(Optimizer):
-    '''Adagrad optimizer.
+    """Adagrad optimizer.
 
     It is recommended to leave the parameters of this optimizer
     at their default values.
@@ -244,16 +271,18 @@ class Adagrad(Optimizer):
     # Arguments
         lr: float >= 0. Learning rate.
         epsilon: float >= 0.
+        decay: float >= 0. Learning rate decay over each update.
 
     # References
         - [Adaptive Subgradient Methods for Online Learning and Stochastic Optimization](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
-    '''
+    """
+
     def __init__(self, lr=0.01, epsilon=1e-8, decay=0., **kwargs):
         super(Adagrad, self).__init__(**kwargs)
-        self.__dict__.update(locals())
         self.lr = K.variable(lr)
+        self.epsilon = epsilon
         self.decay = K.variable(decay)
-        self.inital_decay = decay
+        self.initial_decay = decay
         self.iterations = K.variable(0.)
 
     def get_updates(self, params, constraints, loss):
@@ -264,7 +293,7 @@ class Adagrad(Optimizer):
         self.updates = []
 
         lr = self.lr
-        if self.inital_decay > 0:
+        if self.initial_decay > 0:
             lr *= (1. / (1. + self.decay * self.iterations))
             self.updates.append(K.update_add(self.iterations, 1))
 
@@ -281,13 +310,14 @@ class Adagrad(Optimizer):
 
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adagrad, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
 
 class Adadelta(Optimizer):
-    '''Adadelta optimizer.
+    """Adadelta optimizer.
 
     It is recommended to leave the parameters of this optimizer
     at their default values.
@@ -297,17 +327,20 @@ class Adadelta(Optimizer):
             It is recommended to leave it at the default value.
         rho: float >= 0.
         epsilon: float >= 0. Fuzz factor.
+        decay: float >= 0. Learning rate decay over each update.
 
     # References
         - [Adadelta - an adaptive learning rate method](http://arxiv.org/abs/1212.5701)
-    '''
+    """
+
     def __init__(self, lr=1.0, rho=0.95, epsilon=1e-8, decay=0.,
                  **kwargs):
         super(Adadelta, self).__init__(**kwargs)
-        self.__dict__.update(locals())
         self.lr = K.variable(lr)
+        self.rho = rho
+        self.epsilon = epsilon
         self.decay = K.variable(decay)
-        self.inital_decay = decay
+        self.initial_decay = decay
         self.iterations = K.variable(0.)
 
     def get_updates(self, params, constraints, loss):
@@ -319,7 +352,7 @@ class Adadelta(Optimizer):
         self.updates = []
 
         lr = self.lr
-        if self.inital_decay > 0:
+        if self.initial_decay > 0:
             lr *= (1. / (1. + self.decay * self.iterations))
             self.updates.append(K.update_add(self.iterations, 1))
 
@@ -346,45 +379,50 @@ class Adadelta(Optimizer):
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
                   'rho': self.rho,
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adadelta, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
 
 class Adam(Optimizer):
-    '''Adam optimizer.
+    """Adam optimizer.
 
     Default parameters follow those provided in the original paper.
 
     # Arguments
         lr: float >= 0. Learning rate.
-        beta_1/beta_2: floats, 0 < beta < 1. Generally close to 1.
+        beta_1: float, 0 < beta < 1. Generally close to 1.
+        beta_2: float, 0 < beta < 1. Generally close to 1.
         epsilon: float >= 0. Fuzz factor.
+        decay: float >= 0. Learning rate decay over each update.
 
     # References
         - [Adam - A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980v8)
-    '''
+    """
+
     def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
                  epsilon=1e-8, decay=0., **kwargs):
         super(Adam, self).__init__(**kwargs)
-        self.__dict__.update(locals())
         self.iterations = K.variable(0)
         self.lr = K.variable(lr)
         self.beta_1 = K.variable(beta_1)
         self.beta_2 = K.variable(beta_2)
+        self.epsilon = epsilon
         self.decay = K.variable(decay)
-        self.inital_decay = decay
+        self.initial_decay = decay
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
         self.updates = [K.update_add(self.iterations, 1)]
 
         lr = self.lr
-        if self.inital_decay > 0:
+        if self.initial_decay > 0:
             lr *= (1. / (1. + self.decay * self.iterations))
 
         t = self.iterations + 1
-        lr_t = lr * K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t))
+        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
+                     (1. - K.pow(self.beta_1, t)))
 
         shapes = [K.get_variable_shape(p) for p in params]
         ms = [K.zeros(shape) for shape in shapes]
@@ -411,46 +449,49 @@ class Adam(Optimizer):
         config = {'lr': float(K.get_value(self.lr)),
                   'beta_1': float(K.get_value(self.beta_1)),
                   'beta_2': float(K.get_value(self.beta_2)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adam, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
 
 class Adamax(Optimizer):
-    '''Adamax optimizer from Adam paper's Section 7. It is a variant
-     of Adam based on the infinity norm.
+    """Adamax optimizer from Adam paper's Section 7.
 
+    It is a variant of Adam based on the infinity norm.
     Default parameters follow those provided in the paper.
 
     # Arguments
         lr: float >= 0. Learning rate.
         beta_1/beta_2: floats, 0 < beta < 1. Generally close to 1.
         epsilon: float >= 0. Fuzz factor.
+        decay: float >= 0. Learning rate decay over each update.
 
     # References
         - [Adam - A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980v8)
-    '''
+    """
+
     def __init__(self, lr=0.002, beta_1=0.9, beta_2=0.999,
                  epsilon=1e-8, decay=0., **kwargs):
         super(Adamax, self).__init__(**kwargs)
-        self.__dict__.update(locals())
         self.iterations = K.variable(0.)
         self.lr = K.variable(lr)
         self.beta_1 = K.variable(beta_1)
         self.beta_2 = K.variable(beta_2)
+        self.epsilon = epsilon
         self.decay = K.variable(decay)
-        self.inital_decay = decay
+        self.initial_decay = decay
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
         self.updates = [K.update_add(self.iterations, 1)]
 
         lr = self.lr
-        if self.inital_decay > 0:
+        if self.initial_decay > 0:
             lr *= (1. / (1. + self.decay * self.iterations))
 
         t = self.iterations + 1
-        lr_t = self.lr / (1. - K.pow(self.beta_1, t))
+        lr_t = lr / (1. - K.pow(self.beta_1, t))
 
         shapes = [K.get_variable_shape(p) for p in params]
         # zero init of 1st moment
@@ -480,14 +521,16 @@ class Adamax(Optimizer):
         config = {'lr': float(K.get_value(self.lr)),
                   'beta_1': float(K.get_value(self.beta_1)),
                   'beta_2': float(K.get_value(self.beta_2)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adamax, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
 
 class Nadam(Optimizer):
-    '''
-    Nesterov Adam optimizer: Much like Adam is essentially RMSprop with momentum,
+    """Nesterov Adam optimizer.
+
+    Much like Adam is essentially RMSprop with momentum,
     Nadam is Adam RMSprop with Nesterov momentum.
 
     Default parameters follow those provided in the paper.
@@ -502,16 +545,17 @@ class Nadam(Optimizer):
     # References
         - [Nadam report](http://cs229.stanford.edu/proj2015/054_report.pdf)
         - [On the importance of initialization and momentum in deep learning](http://www.cs.toronto.edu/~fritz/absps/momentum.pdf)
-    '''
+    """
+
     def __init__(self, lr=0.002, beta_1=0.9, beta_2=0.999,
                  epsilon=1e-8, schedule_decay=0.004, **kwargs):
         super(Nadam, self).__init__(**kwargs)
-        self.__dict__.update(locals())
         self.iterations = K.variable(0.)
         self.m_schedule = K.variable(1.)
         self.lr = K.variable(lr)
         self.beta_1 = K.variable(beta_1)
         self.beta_2 = K.variable(beta_2)
+        self.epsilon = epsilon
         self.schedule_decay = schedule_decay
 
     def get_updates(self, params, constraints, loss):
@@ -565,7 +609,40 @@ class Nadam(Optimizer):
         return dict(list(base_config.items()) + list(config.items()))
 
 
-# aliases
+class TFOptimizer(Optimizer):
+    """Wrapper class for native TensorFlow optimizers.
+    """
+
+    def __init__(self, optimizer):
+        self.optimizer = optimizer
+        self.iterations = K.variable(0.)
+        self.updates = []
+
+    def get_updates(self, params, constraints, loss):
+        if constraints:
+            raise ValueError('TF optimizers do not support '
+                             'weights constraints. Either remove '
+                             'all weights constraints in your model, '
+                             'or use a Keras optimizer.')
+        grads = self.optimizer.compute_gradients(loss, params)
+        opt_update = self.optimizer.apply_gradients(
+            grads, global_step=self.iterations)
+        self.updates.append(opt_update)
+        return self.updates
+
+    @property
+    def weights(self):
+        raise NotImplementedError
+
+    def get_config(self):
+        raise NotImplementedError
+
+    def from_config(self, config):
+        raise NotImplementedError
+
+
+# Aliases.
+
 sgd = SGD
 rmsprop = RMSprop
 adagrad = Adagrad
@@ -576,5 +653,10 @@ nadam = Nadam
 
 
 def get(identifier, kwargs=None):
+    if K.backend() == 'tensorflow':
+        # Wrap TF optimizer instances
+        if isinstance(identifier, tf.train.Optimizer):
+            return TFOptimizer(identifier)
+    # Instantiate a Keras optimizer
     return get_from_module(identifier, globals(), 'optimizer',
                            instantiate=True, kwargs=kwargs)

keras/preprocessing/image.py

@@ -1,7 +1,7 @@
-'''Fairly basic set of tools for real-time data augmentation on image data.
+"""Fairly basic set of tools for real-time data augmentation on image data.
 Can easily be extended to include new transformations,
 new preprocessing methods, etc...
-'''
+"""
 from __future__ import absolute_import
 from __future__ import print_function
 
@@ -12,26 +12,67 @@ import scipy.ndimage as ndi
 from six.moves import range
 import os
 import threading
+import warnings
 
 from .. import backend as K
 
+try:
+    from PIL import Image as pil_image
+except ImportError:
+    pil_image = None
 
-def random_rotation(x, rg, row_index=1, col_index=2, channel_index=0,
+
+def random_rotation(x, rg, row_axis=1, col_axis=2, channel_axis=0,
                     fill_mode='nearest', cval=0.):
+    """Performs a random rotation of a Numpy image tensor.
+
+    # Arguments
+        x: Input tensor. Must be 3D.
+        rg: Rotation range, in degrees.
+        row_axis: Index of axis for rows in the input tensor.
+        col_axis: Index of axis for columns in the input tensor.
+        channel_axis: Index of axis for channels in the input tensor.
+        fill_mode: Points outside the boundaries of the input
+            are filled according to the given mode
+            (one of `{'constant', 'nearest', 'reflect', 'wrap'}`).
+        cval: Value used for points outside the boundaries
+            of the input if `mode='constant'`.
+
+    # Returns
+        Rotated Numpy image tensor.
+    """
     theta = np.pi / 180 * np.random.uniform(-rg, rg)
     rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0],
                                 [np.sin(theta), np.cos(theta), 0],
                                 [0, 0, 1]])
 
-    h, w = x.shape[row_index], x.shape[col_index]
+    h, w = x.shape[row_axis], x.shape[col_axis]
     transform_matrix = transform_matrix_offset_center(rotation_matrix, h, w)
-    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
+    x = apply_transform(x, transform_matrix, channel_axis, fill_mode, cval)
     return x
 
 
-def random_shift(x, wrg, hrg, row_index=1, col_index=2, channel_index=0,
+def random_shift(x, wrg, hrg, row_axis=1, col_axis=2, channel_axis=0,
                  fill_mode='nearest', cval=0.):
-    h, w = x.shape[row_index], x.shape[col_index]
+    """Performs a random spatial shift of a Numpy image tensor.
+
+    # Arguments
+        x: Input tensor. Must be 3D.
+        wrg: Width shift range, as a float fraction of the width.
+        hrg: Height shift range, as a float fraction of the height.
+        row_axis: Index of axis for rows in the input tensor.
+        col_axis: Index of axis for columns in the input tensor.
+        channel_axis: Index of axis for channels in the input tensor.
+        fill_mode: Points outside the boundaries of the input
+            are filled according to the given mode
+            (one of `{'constant', 'nearest', 'reflect', 'wrap'}`).
+        cval: Value used for points outside the boundaries
+            of the input if `mode='constant'`.
+
+    # Returns
+        Shifted Numpy image tensor.
+    """
+    h, w = x.shape[row_axis], x.shape[col_axis]
     tx = np.random.uniform(-hrg, hrg) * h
     ty = np.random.uniform(-wrg, wrg) * w
     translation_matrix = np.array([[1, 0, tx],
@@ -39,28 +80,65 @@ def random_shift(x, wrg, hrg, row_index=1, col_index=2, channel_index=0,
                                    [0, 0, 1]])
 
     transform_matrix = translation_matrix  # no need to do offset
-    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
+    x = apply_transform(x, transform_matrix, channel_axis, fill_mode, cval)
     return x
 
 
-def random_shear(x, intensity, row_index=1, col_index=2, channel_index=0,
+def random_shear(x, intensity, row_axis=1, col_axis=2, channel_axis=0,
                  fill_mode='nearest', cval=0.):
+    """Performs a random spatial shear of a Numpy image tensor.
+
+    # Arguments
+        x: Input tensor. Must be 3D.
+        intensity: Transformation intensity.
+        row_axis: Index of axis for rows in the input tensor.
+        col_axis: Index of axis for columns in the input tensor.
+        channel_axis: Index of axis for channels in the input tensor.
+        fill_mode: Points outside the boundaries of the input
+            are filled according to the given mode
+            (one of `{'constant', 'nearest', 'reflect', 'wrap'}`).
+        cval: Value used for points outside the boundaries
+            of the input if `mode='constant'`.
+
+    # Returns
+        Sheared Numpy image tensor.
+    """
     shear = np.random.uniform(-intensity, intensity)
     shear_matrix = np.array([[1, -np.sin(shear), 0],
                              [0, np.cos(shear), 0],
                              [0, 0, 1]])
 
-    h, w = x.shape[row_index], x.shape[col_index]
+    h, w = x.shape[row_axis], x.shape[col_axis]
     transform_matrix = transform_matrix_offset_center(shear_matrix, h, w)
-    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
+    x = apply_transform(x, transform_matrix, channel_axis, fill_mode, cval)
     return x
 
 
-def random_zoom(x, zoom_range, row_index=1, col_index=2, channel_index=0,
+def random_zoom(x, zoom_range, row_axis=1, col_axis=2, channel_axis=0,
                 fill_mode='nearest', cval=0.):
+    """Performs a random spatial zoom of a Numpy image tensor.
+
+    # Arguments
+        x: Input tensor. Must be 3D.
+        zoom_range: Tuple of floats; zoom range for width and height.
+        row_axis: Index of axis for rows in the input tensor.
+        col_axis: Index of axis for columns in the input tensor.
+        channel_axis: Index of axis for channels in the input tensor.
+        fill_mode: Points outside the boundaries of the input
+            are filled according to the given mode
+            (one of `{'constant', 'nearest', 'reflect', 'wrap'}`).
+        cval: Value used for points outside the boundaries
+            of the input if `mode='constant'`.
+
+    # Returns
+        Zoomed Numpy image tensor.
+
+    # Raises
+        ValueError: if `zoom_range` isn't a tuple.
+    """
     if len(zoom_range) != 2:
-        raise Exception('zoom_range should be a tuple or list of two floats. '
-                        'Received arg: ', zoom_range)
+        raise ValueError('zoom_range should be a tuple or list of two floats. '
+                         'Received arg: ', zoom_range)
 
     if zoom_range[0] == 1 and zoom_range[1] == 1:
         zx, zy = 1, 1
@@ -70,24 +148,19 @@ def random_zoom(x, zoom_range, row_index=1, col_index=2, channel_index=0,
                             [0, zy, 0],
                             [0, 0, 1]])
 
-    h, w = x.shape[row_index], x.shape[col_index]
+    h, w = x.shape[row_axis], x.shape[col_axis]
     transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w)
-    x = apply_transform(x, transform_matrix, channel_index, fill_mode, cval)
+    x = apply_transform(x, transform_matrix, channel_axis, fill_mode, cval)
     return x
 
 
-def random_barrel_transform(x, intensity):
-    # TODO
-    pass
-
-
-def random_channel_shift(x, intensity, channel_index=0):
-    x = np.rollaxis(x, channel_index, 0)
+def random_channel_shift(x, intensity, channel_axis=0):
+    x = np.rollaxis(x, channel_axis, 0)
     min_x, max_x = np.min(x), np.max(x)
     channel_images = [np.clip(x_channel + np.random.uniform(-intensity, intensity), min_x, max_x)
                       for x_channel in x]
     x = np.stack(channel_images, axis=0)
-    x = np.rollaxis(x, 0, channel_index+1)
+    x = np.rollaxis(x, 0, channel_axis + 1)
     return x
 
 
@@ -100,14 +173,14 @@ def transform_matrix_offset_center(matrix, x, y):
     return transform_matrix
 
 
-def apply_transform(x, transform_matrix, channel_index=0, fill_mode='nearest', cval=0.):
-    x = np.rollaxis(x, channel_index, 0)
+def apply_transform(x, transform_matrix, channel_axis=0, fill_mode='nearest', cval=0.):
+    x = np.rollaxis(x, channel_axis, 0)
     final_affine_matrix = transform_matrix[:2, :2]
     final_offset = transform_matrix[:2, 2]
     channel_images = [ndi.interpolation.affine_transform(x_channel, final_affine_matrix,
-                      final_offset, order=0, mode=fill_mode, cval=cval) for x_channel in x]
+                                                         final_offset, order=0, mode=fill_mode, cval=cval) for x_channel in x]
     x = np.stack(channel_images, axis=0)
-    x = np.rollaxis(x, 0, channel_index+1)
+    x = np.rollaxis(x, 0, channel_axis + 1)
     return x
 
 
@@ -119,34 +192,76 @@ def flip_axis(x, axis):
 
 
 def array_to_img(x, dim_ordering='default', scale=True):
-    from PIL import Image
+    """Converts a 3D Numpy array to a PIL Image instance.
+
+    # Arguments
+        x: Input Numpy array.
+        dim_ordering: Image data format.
+        scale: Whether to rescale image values
+            to be within [0, 255].
+
+    # Returns
+        A PIL Image instance.
+
+    # Raises
+        ImportError: if PIL is not available.
+        ValueError: if invalid `x` or `dim_ordering` is passed.
+    """
+    if pil_image is None:
+        raise ImportError('Could not import PIL.Image. '
+                          'The use of `array_to_img` requires PIL.')
+    x = np.asarray(x, dtype=K.floatx())
+    if x.ndim != 3:
+        raise ValueError('Expected image array to have rank 3 (single image). '
+                         'Got array with shape:', x.shape)
+
     if dim_ordering == 'default':
         dim_ordering = K.image_dim_ordering()
+    if dim_ordering not in {'th', 'tf'}:
+        raise ValueError('Invalid dim_ordering:', dim_ordering)
+
+    # Original Numpy array x has format (height, width, channel)
+    # or (channel, height, width)
+    # but target PIL image has format (width, height, channel)
     if dim_ordering == 'th':
         x = x.transpose(1, 2, 0)
     if scale:
-        x += max(-np.min(x), 0)
+        x = x + max(-np.min(x), 0)
         x_max = np.max(x)
         if x_max != 0:
             x /= x_max
         x *= 255
     if x.shape[2] == 3:
         # RGB
-        return Image.fromarray(x.astype('uint8'), 'RGB')
+        return pil_image.fromarray(x.astype('uint8'), 'RGB')
     elif x.shape[2] == 1:
         # grayscale
-        return Image.fromarray(x[:, :, 0].astype('uint8'), 'L')
+        return pil_image.fromarray(x[:, :, 0].astype('uint8'), 'L')
     else:
-        raise Exception('Unsupported channel number: ', x.shape[2])
+        raise ValueError('Unsupported channel number: ', x.shape[2])
 
 
 def img_to_array(img, dim_ordering='default'):
+    """Converts a PIL Image instance to a Numpy array.
+
+    # Arguments
+        img: PIL Image instance.
+        dim_ordering: Image data format.
+
+    # Returns
+        A 3D Numpy array.
+
+    # Raises
+        ValueError: if invalid `img` or `dim_ordering` is passed.
+    """
     if dim_ordering == 'default':
         dim_ordering = K.image_dim_ordering()
-    if dim_ordering not in ['th', 'tf']:
-        raise Exception('Unknown dim_ordering: ', dim_ordering)
-    # image has dim_ordering (height, width, channel)
-    x = np.asarray(img, dtype='float32')
+    if dim_ordering not in {'th', 'tf'}:
+        raise ValueError('Unknown dim_ordering: ', dim_ordering)
+    # Numpy array x has format (height, width, channel)
+    # or (channel, height, width)
+    # but original PIL image has format (width, height, channel)
+    x = np.asarray(img, dtype=K.floatx())
     if len(x.shape) == 3:
         if dim_ordering == 'th':
             x = x.transpose(2, 0, 1)
@@ -156,21 +271,29 @@ def img_to_array(img, dim_ordering='default'):
         else:
             x = x.reshape((x.shape[0], x.shape[1], 1))
     else:
-        raise Exception('Unsupported image shape: ', x.shape)
+        raise ValueError('Unsupported image shape: ', x.shape)
     return x
 
 
 def load_img(path, grayscale=False, target_size=None):
-    '''Load an image into PIL format.
+    """Loads an image into PIL format.
 
     # Arguments
-        path: path to image file
-        grayscale: boolean
-        target_size: None (default to original size)
-            or (img_height, img_width)
-    '''
-    from PIL import Image
-    img = Image.open(path)
+        path: Path to image file
+        grayscale: Boolean, whether to load the image as grayscale.
+        target_size: Either `None` (default to original size)
+            or tuple of ints `(img_height, img_width)`.
+
+    # Returns
+        A PIL Image instance.
+
+    # Raises
+        ImportError: if PIL is not available.
+    """
+    if pil_image is None:
+        raise ImportError('Could not import PIL.Image. '
+                          'The use of `array_to_img` requires PIL.')
+    img = pil_image.open(path)
     if grayscale:
         img = img.convert('L')
     else:  # Ensure 3 channel even when loaded image is grayscale
@@ -181,13 +304,13 @@ def load_img(path, grayscale=False, target_size=None):
 
 
 def list_pictures(directory, ext='jpg|jpeg|bmp|png'):
-    return [os.path.join(directory, f) for f in os.listdir(directory)
-            if os.path.isfile(os.path.join(directory, f)) and re.match('([\w]+\.(?:' + ext + '))', f)]
+    return [os.path.join(root, f)
+            for root, _, files in os.walk(directory) for f in files
+            if re.match('([\w]+\.(?:' + ext + '))', f)]
 
 
 class ImageDataGenerator(object):
-    '''Generate minibatches with
-    real-time data augmentation.
+    """Generate minibatches of image data with real-time data augmentation.
 
     # Arguments
         featurewise_center: set input mean to 0 over the dataset.
@@ -211,14 +334,20 @@ class ImageDataGenerator(object):
         horizontal_flip: whether to randomly flip images horizontally.
         vertical_flip: whether to randomly flip images vertically.
         rescale: rescaling factor. If None or 0, no rescaling is applied,
-            otherwise we multiply the data by the value provided (before applying
-            any other transformation).
+            otherwise we multiply the data by the value provided
+            (before applying any other transformation).
+        preprocessing_function: function that will be implied on each input.
+            The function will run before any other modification on it.
+            The function should take one argument:
+            one image (Numpy tensor with rank 3),
+            and should output a Numpy tensor with the same shape.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode it is at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
-    '''
+            If you never set it, then it will be "tf".
+    """
+
     def __init__(self,
                  featurewise_center=False,
                  samplewise_center=False,
@@ -236,88 +365,134 @@ class ImageDataGenerator(object):
                  horizontal_flip=False,
                  vertical_flip=False,
                  rescale=None,
+                 preprocessing_function=None,
                  dim_ordering='default'):
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
-        self.__dict__.update(locals())
+        self.featurewise_center = featurewise_center
+        self.samplewise_center = samplewise_center
+        self.featurewise_std_normalization = featurewise_std_normalization
+        self.samplewise_std_normalization = samplewise_std_normalization
+        self.zca_whitening = zca_whitening
+        self.rotation_range = rotation_range
+        self.width_shift_range = width_shift_range
+        self.height_shift_range = height_shift_range
+        self.shear_range = shear_range
+        self.zoom_range = zoom_range
+        self.channel_shift_range = channel_shift_range
+        self.fill_mode = fill_mode
+        self.cval = cval
+        self.horizontal_flip = horizontal_flip
+        self.vertical_flip = vertical_flip
+        self.rescale = rescale
+        self.preprocessing_function = preprocessing_function
+
+        if dim_ordering not in {'tf', 'th'}:
+            raise ValueError('dim_ordering should be "tf" (channel after row and '
+                             'column) or "th" (channel before row and column). '
+                             'Received arg: ', dim_ordering)
+        self.dim_ordering = dim_ordering
+        if dim_ordering == 'th':
+            self.channel_axis = 1
+            self.row_axis = 2
+            self.col_axis = 3
+        if dim_ordering == 'tf':
+            self.channel_axis = 3
+            self.row_axis = 1
+            self.col_axis = 2
+
         self.mean = None
         self.std = None
         self.principal_components = None
-        self.rescale = rescale
-
-        if dim_ordering not in {'tf', 'th'}:
-            raise Exception('dim_ordering should be "tf" (channel after row and '
-                            'column) or "th" (channel before row and column). '
-                            'Received arg: ', dim_ordering)
-        self.dim_ordering = dim_ordering
-        if dim_ordering == 'th':
-            self.channel_index = 1
-            self.row_index = 2
-            self.col_index = 3
-        if dim_ordering == 'tf':
-            self.channel_index = 3
-            self.row_index = 1
-            self.col_index = 2
 
         if np.isscalar(zoom_range):
             self.zoom_range = [1 - zoom_range, 1 + zoom_range]
         elif len(zoom_range) == 2:
             self.zoom_range = [zoom_range[0], zoom_range[1]]
         else:
-            raise Exception('zoom_range should be a float or '
-                            'a tuple or list of two floats. '
-                            'Received arg: ', zoom_range)
+            raise ValueError('zoom_range should be a float or '
+                             'a tuple or list of two floats. '
+                             'Received arg: ', zoom_range)
 
     def flow(self, X, y=None, batch_size=32, shuffle=True, seed=None,
              save_to_dir=None, save_prefix='', save_format='jpeg'):
         return NumpyArrayIterator(
             X, y, self,
-            batch_size=batch_size, shuffle=shuffle, seed=seed,
+            batch_size=batch_size,
+            shuffle=shuffle,
+            seed=seed,
             dim_ordering=self.dim_ordering,
-            save_to_dir=save_to_dir, save_prefix=save_prefix, save_format=save_format)
+            save_to_dir=save_to_dir,
+            save_prefix=save_prefix,
+            save_format=save_format)
 
     def flow_from_directory(self, directory,
                             target_size=(256, 256), color_mode='rgb',
                             classes=None, class_mode='categorical',
                             batch_size=32, shuffle=True, seed=None,
-                            save_to_dir=None, save_prefix='', save_format='jpeg'):
+                            save_to_dir=None,
+                            save_prefix='',
+                            save_format='jpeg',
+                            follow_links=False):
         return DirectoryIterator(
             directory, self,
             target_size=target_size, color_mode=color_mode,
             classes=classes, class_mode=class_mode,
             dim_ordering=self.dim_ordering,
             batch_size=batch_size, shuffle=shuffle, seed=seed,
-            save_to_dir=save_to_dir, save_prefix=save_prefix, save_format=save_format)
+            save_to_dir=save_to_dir,
+            save_prefix=save_prefix,
+            save_format=save_format,
+            follow_links=follow_links)
 
     def standardize(self, x):
+        if self.preprocessing_function:
+            x = self.preprocessing_function(x)
         if self.rescale:
             x *= self.rescale
         # x is a single image, so it doesn't have image number at index 0
-        img_channel_index = self.channel_index - 1
+        img_channel_axis = self.channel_axis - 1
         if self.samplewise_center:
-            x -= np.mean(x, axis=img_channel_index, keepdims=True)
+            x -= np.mean(x, axis=img_channel_axis, keepdims=True)
         if self.samplewise_std_normalization:
-            x /= (np.std(x, axis=img_channel_index, keepdims=True) + 1e-7)
+            x /= (np.std(x, axis=img_channel_axis, keepdims=True) + 1e-7)
 
         if self.featurewise_center:
-            x -= self.mean
+            if self.mean is not None:
+                x -= self.mean
+            else:
+                warnings.warn('This ImageDataGenerator specifies '
+                              '`featurewise_center`, but it hasn\'t'
+                              'been fit on any training data. Fit it '
+                              'first by calling `.fit(numpy_data)`.')
         if self.featurewise_std_normalization:
-            x /= (self.std + 1e-7)
-
+            if self.std is not None:
+                x /= (self.std + 1e-7)
+            else:
+                warnings.warn('This ImageDataGenerator specifies '
+                              '`featurewise_std_normalization`, but it hasn\'t'
+                              'been fit on any training data. Fit it '
+                              'first by calling `.fit(numpy_data)`.')
         if self.zca_whitening:
-            flatx = np.reshape(x, (x.size))
-            whitex = np.dot(flatx, self.principal_components)
-            x = np.reshape(whitex, (x.shape[0], x.shape[1], x.shape[2]))
-
+            if self.principal_components is not None:
+                flatx = np.reshape(x, (x.size))
+                whitex = np.dot(flatx, self.principal_components)
+                x = np.reshape(whitex, (x.shape[0], x.shape[1], x.shape[2]))
+            else:
+                warnings.warn('This ImageDataGenerator specifies '
+                              '`zca_whitening`, but it hasn\'t'
+                              'been fit on any training data. Fit it '
+                              'first by calling `.fit(numpy_data)`.')
         return x
 
     def random_transform(self, x):
         # x is a single image, so it doesn't have image number at index 0
-        img_row_index = self.row_index - 1
-        img_col_index = self.col_index - 1
-        img_channel_index = self.channel_index - 1
+        img_row_axis = self.row_axis - 1
+        img_col_axis = self.col_axis - 1
+        img_channel_axis = self.channel_axis - 1
 
-        # use composition of homographies to generate final transform that needs to be applied
+        # use composition of homographies
+        # to generate final transform that needs to be applied
         if self.rotation_range:
             theta = np.pi / 180 * np.random.uniform(-self.rotation_range, self.rotation_range)
         else:
@@ -326,12 +501,12 @@ class ImageDataGenerator(object):
                                     [np.sin(theta), np.cos(theta), 0],
                                     [0, 0, 1]])
         if self.height_shift_range:
-            tx = np.random.uniform(-self.height_shift_range, self.height_shift_range) * x.shape[img_row_index]
+            tx = np.random.uniform(-self.height_shift_range, self.height_shift_range) * x.shape[img_row_axis]
         else:
             tx = 0
 
         if self.width_shift_range:
-            ty = np.random.uniform(-self.width_shift_range, self.width_shift_range) * x.shape[img_col_index]
+            ty = np.random.uniform(-self.width_shift_range, self.width_shift_range) * x.shape[img_col_axis]
         else:
             ty = 0
 
@@ -354,96 +529,125 @@ class ImageDataGenerator(object):
                                 [0, zy, 0],
                                 [0, 0, 1]])
 
-        transform_matrix = np.dot(np.dot(np.dot(rotation_matrix, translation_matrix), shear_matrix), zoom_matrix)
+        transform_matrix = np.dot(np.dot(np.dot(rotation_matrix,
+                                                translation_matrix),
+                                         shear_matrix),
+                                  zoom_matrix)
 
-        h, w = x.shape[img_row_index], x.shape[img_col_index]
+        h, w = x.shape[img_row_axis], x.shape[img_col_axis]
         transform_matrix = transform_matrix_offset_center(transform_matrix, h, w)
-        x = apply_transform(x, transform_matrix, img_channel_index,
+        x = apply_transform(x, transform_matrix, img_channel_axis,
                             fill_mode=self.fill_mode, cval=self.cval)
         if self.channel_shift_range != 0:
-            x = random_channel_shift(x, self.channel_shift_range, img_channel_index)
-
+            x = random_channel_shift(x,
+                                     self.channel_shift_range,
+                                     img_channel_axis)
         if self.horizontal_flip:
             if np.random.random() < 0.5:
-                x = flip_axis(x, img_col_index)
+                x = flip_axis(x, img_col_axis)
 
         if self.vertical_flip:
             if np.random.random() < 0.5:
-                x = flip_axis(x, img_row_index)
+                x = flip_axis(x, img_row_axis)
 
-        # TODO:
-        # channel-wise normalization
-        # barrel/fisheye
         return x
 
-    def fit(self, X,
+    def fit(self, x,
             augment=False,
             rounds=1,
             seed=None):
-        '''Required for featurewise_center, featurewise_std_normalization
+        """Required for featurewise_center, featurewise_std_normalization
         and zca_whitening.
 
         # Arguments
-            X: Numpy array, the data to fit on.
-            augment: whether to fit on randomly augmented samples
-            rounds: if `augment`,
+            x: Numpy array, the data to fit on. Should have rank 4.
+                In case of grayscale data,
+                the channels axis should have value 1, and in case
+                of RGB data, it should have value 3.
+            augment: Whether to fit on randomly augmented samples
+            rounds: If `augment`,
                 how many augmentation passes to do over the data
             seed: random seed.
-        '''
-        X = np.copy(X)
+
+        # Raises
+            ValueError: in case of invalid input `x`.
+        """
+        x = np.asarray(x, dtype=K.floatx())
+        if x.ndim != 4:
+            raise ValueError('Input to `.fit()` should have rank 4. '
+                             'Got array with shape: ' + str(x.shape))
+        if x.shape[self.channel_axis] not in {1, 3, 4}:
+            raise ValueError(
+                'Expected input to be images (as Numpy array) '
+                'following the dimension ordering convention "' + self.dim_ordering + '" '
+                '(channels on axis ' + str(self.channel_axis) + '), i.e. expected '
+                'either 1, 3 or 4 channels on axis ' + str(self.channel_axis) + '. '
+                'However, it was passed an array with shape ' + str(x.shape) +
+                ' (' + str(x.shape[self.channel_axis]) + ' channels).')
+
+        if seed is not None:
+            np.random.seed(seed)
+
+        x = np.copy(x)
         if augment:
-            aX = np.zeros(tuple([rounds * X.shape[0]] + list(X.shape)[1:]))
+            ax = np.zeros(tuple([rounds * x.shape[0]] + list(x.shape)[1:]), dtype=K.floatx())
             for r in range(rounds):
-                for i in range(X.shape[0]):
-                    aX[i + r * X.shape[0]] = self.random_transform(X[i])
-            X = aX
+                for i in range(x.shape[0]):
+                    ax[i + r * x.shape[0]] = self.random_transform(x[i])
+            x = ax
 
         if self.featurewise_center:
-            self.mean = np.mean(X, axis=0)
-            X -= self.mean
+            self.mean = np.mean(x, axis=(0, self.row_axis, self.col_axis))
+            broadcast_shape = [1, 1, 1]
+            broadcast_shape[self.channel_axis - 1] = x.shape[self.channel_axis]
+            self.mean = np.reshape(self.mean, broadcast_shape)
+            x -= self.mean
 
         if self.featurewise_std_normalization:
-            self.std = np.std(X, axis=0)
-            X /= (self.std + 1e-7)
+            self.std = np.std(x, axis=(0, self.row_axis, self.col_axis))
+            broadcast_shape = [1, 1, 1]
+            broadcast_shape[self.channel_axis - 1] = x.shape[self.channel_axis]
+            self.std = np.reshape(self.std, broadcast_shape)
+            x /= (self.std + K.epsilon())
 
         if self.zca_whitening:
-            flatX = np.reshape(X, (X.shape[0], X.shape[1] * X.shape[2] * X.shape[3]))
-            sigma = np.dot(flatX.T, flatX) / flatX.shape[1]
-            U, S, V = linalg.svd(sigma)
-            self.principal_components = np.dot(np.dot(U, np.diag(1. / np.sqrt(S + 10e-7))), U.T)
+            flat_x = np.reshape(x, (x.shape[0], x.shape[1] * x.shape[2] * x.shape[3]))
+            sigma = np.dot(flat_x.T, flat_x) / flat_x.shape[0]
+            u, s, _ = linalg.svd(sigma)
+            self.principal_components = np.dot(np.dot(u, np.diag(1. / np.sqrt(s + 10e-7))), u.T)
 
 
 class Iterator(object):
 
-    def __init__(self, N, batch_size, shuffle, seed):
-        self.N = N
+    def __init__(self, n, batch_size, shuffle, seed):
+        self.n = n
         self.batch_size = batch_size
         self.shuffle = shuffle
         self.batch_index = 0
         self.total_batches_seen = 0
         self.lock = threading.Lock()
-        self.index_generator = self._flow_index(N, batch_size, shuffle, seed)
+        self.index_generator = self._flow_index(n, batch_size, shuffle, seed)
 
     def reset(self):
         self.batch_index = 0
 
-    def _flow_index(self, N, batch_size=32, shuffle=False, seed=None):
+    def _flow_index(self, n, batch_size=32, shuffle=False, seed=None):
         # ensure self.batch_index is 0
         self.reset()
         while 1:
+            if seed is not None:
+                np.random.seed(seed + self.total_batches_seen)
             if self.batch_index == 0:
-                index_array = np.arange(N)
+                index_array = np.arange(n)
                 if shuffle:
-                    if seed is not None:
-                        np.random.seed(seed + self.total_batches_seen)
-                    index_array = np.random.permutation(N)
+                    index_array = np.random.permutation(n)
 
-            current_index = (self.batch_index * batch_size) % N
-            if N >= current_index + batch_size:
+            current_index = (self.batch_index * batch_size) % n
+            if n >= current_index + batch_size:
                 current_batch_size = batch_size
                 self.batch_index += 1
             else:
-                current_batch_size = N - current_index
+                current_batch_size = n - current_index
                 self.batch_index = 0
             self.total_batches_seen += 1
             yield (index_array[current_index: current_index + current_batch_size],
@@ -460,24 +664,40 @@ class Iterator(object):
 
 class NumpyArrayIterator(Iterator):
 
-    def __init__(self, X, y, image_data_generator,
+    def __init__(self, x, y, image_data_generator,
                  batch_size=32, shuffle=False, seed=None,
                  dim_ordering='default',
                  save_to_dir=None, save_prefix='', save_format='jpeg'):
-        if y is not None and len(X) != len(y):
-            raise Exception('X (images tensor) and y (labels) '
-                            'should have the same length. '
-                            'Found: X.shape = %s, y.shape = %s' % (np.asarray(X).shape, np.asarray(y).shape))
+        if y is not None and len(x) != len(y):
+            raise ValueError('X (images tensor) and y (labels) '
+                             'should have the same length. '
+                             'Found: X.shape = %s, y.shape = %s' %
+                             (np.asarray(x).shape, np.asarray(y).shape))
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
-        self.X = X
-        self.y = y
+        self.x = np.asarray(x, dtype=K.floatx())
+        if self.x.ndim != 4:
+            raise ValueError('Input data in `NumpyArrayIterator` '
+                             'should have rank 4. You passed an array '
+                             'with shape', self.x.shape)
+        channels_axis = 3 if dim_ordering == 'tf' else 1
+        if self.x.shape[channels_axis] not in {1, 3, 4}:
+            raise ValueError('NumpyArrayIterator is set to use the '
+                             'dimension ordering convention "' + dim_ordering + '" '
+                             '(channels on axis ' + str(channels_axis) + '), i.e. expected '
+                             'either 1, 3 or 4 channels on axis ' + str(channels_axis) + '. '
+                             'However, it was passed an array with shape ' + str(self.x.shape) +
+                             ' (' + str(self.x.shape[channels_axis]) + ' channels).')
+        if y is not None:
+            self.y = np.asarray(y)
+        else:
+            self.y = None
         self.image_data_generator = image_data_generator
         self.dim_ordering = dim_ordering
         self.save_to_dir = save_to_dir
         self.save_prefix = save_prefix
         self.save_format = save_format
-        super(NumpyArrayIterator, self).__init__(X.shape[0], batch_size, shuffle, seed)
+        super(NumpyArrayIterator, self).__init__(x.shape[0], batch_size, shuffle, seed)
 
     def next(self):
         # for python 2.x.
@@ -486,11 +706,12 @@ class NumpyArrayIterator(Iterator):
         # see http://anandology.com/blog/using-iterators-and-generators/
         with self.lock:
             index_array, current_index, current_batch_size = next(self.index_generator)
-        # The transformation of images is not under thread lock so it can be done in parallel
-        batch_x = np.zeros(tuple([current_batch_size] + list(self.X.shape)[1:]))
+        # The transformation of images is not under thread lock
+        # so it can be done in parallel
+        batch_x = np.zeros(tuple([current_batch_size] + list(self.x.shape)[1:]), dtype=K.floatx())
         for i, j in enumerate(index_array):
-            x = self.X[j]
-            x = self.image_data_generator.random_transform(x.astype('float32'))
+            x = self.x[j]
+            x = self.image_data_generator.random_transform(x.astype(K.floatx()))
             x = self.image_data_generator.standardize(x)
             batch_x[i] = x
         if self.save_to_dir:
@@ -514,7 +735,8 @@ class DirectoryIterator(Iterator):
                  dim_ordering='default',
                  classes=None, class_mode='categorical',
                  batch_size=32, shuffle=True, seed=None,
-                 save_to_dir=None, save_prefix='', save_format='jpeg'):
+                 save_to_dir=None, save_prefix='', save_format='jpeg',
+                 follow_links=False):
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
         self.directory = directory
@@ -558,16 +780,20 @@ class DirectoryIterator(Iterator):
         self.nb_class = len(classes)
         self.class_indices = dict(zip(classes, range(len(classes))))
 
+        def _recursive_list(subpath):
+            return sorted(os.walk(subpath, followlinks=follow_links), key=lambda tpl: tpl[0])
+
         for subdir in classes:
             subpath = os.path.join(directory, subdir)
-            for fname in os.listdir(subpath):
-                is_valid = False
-                for extension in white_list_formats:
-                    if fname.lower().endswith('.' + extension):
-                        is_valid = True
-                        break
-                if is_valid:
-                    self.nb_sample += 1
+            for root, _, files in _recursive_list(subpath):
+                for fname in files:
+                    is_valid = False
+                    for extension in white_list_formats:
+                        if fname.lower().endswith('.' + extension):
+                            is_valid = True
+                            break
+                    if is_valid:
+                        self.nb_sample += 1
         print('Found %d images belonging to %d classes.' % (self.nb_sample, self.nb_class))
 
         # second, build an index of the images in the different class subfolders
@@ -576,28 +802,34 @@ class DirectoryIterator(Iterator):
         i = 0
         for subdir in classes:
             subpath = os.path.join(directory, subdir)
-            for fname in os.listdir(subpath):
-                is_valid = False
-                for extension in white_list_formats:
-                    if fname.lower().endswith('.' + extension):
-                        is_valid = True
-                        break
-                if is_valid:
-                    self.classes[i] = self.class_indices[subdir]
-                    self.filenames.append(os.path.join(subdir, fname))
-                    i += 1
+            for root, _, files in _recursive_list(subpath):
+                for fname in files:
+                    is_valid = False
+                    for extension in white_list_formats:
+                        if fname.lower().endswith('.' + extension):
+                            is_valid = True
+                            break
+                    if is_valid:
+                        self.classes[i] = self.class_indices[subdir]
+                        i += 1
+                        # add filename relative to directory
+                        absolute_path = os.path.join(root, fname)
+                        self.filenames.append(os.path.relpath(absolute_path, directory))
         super(DirectoryIterator, self).__init__(self.nb_sample, batch_size, shuffle, seed)
 
     def next(self):
         with self.lock:
             index_array, current_index, current_batch_size = next(self.index_generator)
-        # The transformation of images is not under thread lock so it can be done in parallel
-        batch_x = np.zeros((current_batch_size,) + self.image_shape)
+        # The transformation of images is not under thread lock
+        # so it can be done in parallel
+        batch_x = np.zeros((current_batch_size,) + self.image_shape, dtype=K.floatx())
         grayscale = self.color_mode == 'grayscale'
         # build batch of image data
         for i, j in enumerate(index_array):
             fname = self.filenames[j]
-            img = load_img(os.path.join(self.directory, fname), grayscale=grayscale, target_size=self.target_size)
+            img = load_img(os.path.join(self.directory, fname),
+                           grayscale=grayscale,
+                           target_size=self.target_size)
             x = img_to_array(img, dim_ordering=self.dim_ordering)
             x = self.image_data_generator.random_transform(x)
             x = self.image_data_generator.standardize(x)
@@ -615,9 +847,9 @@ class DirectoryIterator(Iterator):
         if self.class_mode == 'sparse':
             batch_y = self.classes[index_array]
         elif self.class_mode == 'binary':
-            batch_y = self.classes[index_array].astype('float32')
+            batch_y = self.classes[index_array].astype(K.floatx())
         elif self.class_mode == 'categorical':
-            batch_y = np.zeros((len(batch_x), self.nb_class), dtype='float32')
+            batch_y = np.zeros((len(batch_x), self.nb_class), dtype=K.floatx())
             for i, label in enumerate(self.classes[index_array]):
                 batch_y[i, label] = 1.
         else:

keras/preprocessing/sequence.py

@@ -1,5 +1,6 @@
-from __future__ import absolute_import
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
 import numpy as np
 import random
 from six.moves import range
@@ -7,8 +8,7 @@ from six.moves import range
 
 def pad_sequences(sequences, maxlen=None, dtype='int32',
                   padding='pre', truncating='pre', value=0.):
-    '''Pads each sequence to the same length:
-    the length of the longest sequence.
+    """Pads each sequence to the same length (length of the longest sequence).
 
     If maxlen is provided, any sequence longer
     than maxlen is truncated to maxlen.
@@ -28,7 +28,11 @@ def pad_sequences(sequences, maxlen=None, dtype='int32',
 
     # Returns
         x: numpy array with dimensions (number_of_sequences, maxlen)
-    '''
+
+    # Raises
+        ValueError: in case of invalid values for `truncating` or `padding`,
+            or in case of invalid shape for a `sequences` entry.
+    """
     lengths = [len(s) for s in sequences]
 
     nb_samples = len(sequences)
@@ -45,8 +49,8 @@ def pad_sequences(sequences, maxlen=None, dtype='int32',
 
     x = (np.ones((nb_samples, maxlen) + sample_shape) * value).astype(dtype)
     for idx, s in enumerate(sequences):
-        if len(s) == 0:
-            continue  # empty list was found
+        if not len(s):
+            continue  # empty list/array was found
         if truncating == 'pre':
             trunc = s[-maxlen:]
         elif truncating == 'post':
@@ -70,7 +74,9 @@ def pad_sequences(sequences, maxlen=None, dtype='int32',
 
 
 def make_sampling_table(size, sampling_factor=1e-5):
-    '''This generates an array where the ith element
+    """Generates a word rank-based probabilistic sampling table.
+
+    This generates an array where the ith element
     is the probability that a word of rank i would be sampled,
     according to the sampling distribution used in word2vec.
 
@@ -84,11 +90,16 @@ def make_sampling_table(size, sampling_factor=1e-5):
 
     # Arguments
         size: int, number of possible words to sample.
-    '''
+        sampling_factor: the sampling factor in the word2vec formula.
+
+    # Returns
+        A 1D Numpy array of length `size` where the ith entry
+        is the probability that a word of rank i should be sampled.
+    """
     gamma = 0.577
     rank = np.array(list(range(size)))
     rank[0] = 1
-    inv_fq = rank * (np.log(rank) + gamma) + 0.5 - 1./(12.*rank)
+    inv_fq = rank * (np.log(rank) + gamma) + 0.5 - 1. / (12. * rank)
     f = sampling_factor * inv_fq
 
     return np.minimum(1., f / np.sqrt(f))
@@ -97,29 +108,40 @@ def make_sampling_table(size, sampling_factor=1e-5):
 def skipgrams(sequence, vocabulary_size,
               window_size=4, negative_samples=1., shuffle=True,
               categorical=False, sampling_table=None):
-    '''Take a sequence (list of indexes of words),
+    """Generates skipgram word pairs.
+
+    Takes a sequence (list of indexes of words),
     returns couples of [word_index, other_word index] and labels (1s or 0s),
     where label = 1 if 'other_word' belongs to the context of 'word',
     and label=0 if 'other_word' is randomly sampled
 
     # Arguments
+        sequence: a word sequence (sentence), encoded as a list
+            of word indices (integers). If using a `sampling_table`,
+            word indices are expected to match the rank
+            of the words in a reference dataset (e.g. 10 would encode
+            the 10-th most frequently occuring token).
+            Note that index 0 is expected to be a non-word and will be skipped.
         vocabulary_size: int. maximum possible word index + 1
         window_size: int. actually half-window.
             The window of a word wi will be [i-window_size, i+window_size+1]
         negative_samples: float >= 0. 0 for no negative (=random) samples.
             1 for same number as positive samples. etc.
+        shuffle: whether to shuffle the word couples before returning them.
         categorical: bool. if False, labels will be
             integers (eg. [0, 1, 1 .. ]),
             if True labels will be categorical eg. [[1,0],[0,1],[0,1] .. ]
+        sampling_table: 1D array of size `vocabulary_size` where the entry i
+            encodes the probabibily to sample a word of rank i.
 
     # Returns
         couples, labels: where `couples` are int pairs and
             `labels` are either 0 or 1.
 
-    # Notes
+    # Note
         By convention, index 0 in the vocabulary is
         a non-word and will be skipped.
-    '''
+    """
     couples = []
     labels = []
     for i, wi in enumerate(sequence):
@@ -129,8 +151,8 @@ def skipgrams(sequence, vocabulary_size,
             if sampling_table[wi] < random.random():
                 continue
 
-        window_start = max(0, i-window_size)
-        window_end = min(len(sequence), i+window_size+1)
+        window_start = max(0, i - window_size)
+        window_end = min(len(sequence), i + window_size + 1)
         for j in range(window_start, window_end):
             if j != i:
                 wj = sequence[j]
@@ -147,11 +169,12 @@ def skipgrams(sequence, vocabulary_size,
         words = [c[0] for c in couples]
         random.shuffle(words)
 
-        couples += [[words[i %len(words)], random.randint(1, vocabulary_size-1)] for i in range(nb_negative_samples)]
+        couples += [[words[i % len(words)],
+                    random.randint(1, vocabulary_size - 1)] for i in range(nb_negative_samples)]
         if categorical:
-            labels += [[1, 0]]*nb_negative_samples
+            labels += [[1, 0]] * nb_negative_samples
         else:
-            labels += [0]*nb_negative_samples
+            labels += [0] * nb_negative_samples
 
     if shuffle:
         seed = random.randint(0, 10e6)

keras/preprocessing/text.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
-'''These preprocessing utilities would greatly benefit
-from a fast Cython rewrite.
-'''
+"""Utilities for text input preprocessing.
+
+May benefit from a fast Cython rewrite.
+"""
 from __future__ import absolute_import
 from __future__ import division
 
@@ -17,54 +18,70 @@ else:
     maketrans = str.maketrans
 
 
-def base_filter():
-    f = string.punctuation
-    f = f.replace("'", '')
-    f += '\t\n'
-    return f
+def text_to_word_sequence(text,
+                          filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
+                          lower=True, split=" "):
+    """Converts a text to a sequence of word indices.
 
+    # Arguments
+        text: Input text (string).
+        filters: Sequence of characters to filter out.
+        lower: Whether to convert the input to lowercase.
+        split: Sentence split marker (string).
 
-def text_to_word_sequence(text, filters=base_filter(), lower=True, split=" "):
-    '''prune: sequence of characters to filter out
-    '''
+    # Returns
+        A list of integer word indices.
+    """
     if lower:
         text = text.lower()
-    text = text.translate(maketrans(filters, split*len(filters)))
+    text = text.translate(maketrans(filters, split * len(filters)))
     seq = text.split(split)
-    return [_f for _f in seq if _f]
+    return [i for i in seq if i]
 
 
-def one_hot(text, n, filters=base_filter(), lower=True, split=" "):
-    seq = text_to_word_sequence(text, filters=filters, lower=lower, split=split)
+def one_hot(text, n,
+            filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
+            lower=True,
+            split=' '):
+    seq = text_to_word_sequence(text,
+                                filters=filters,
+                                lower=lower,
+                                split=split)
     return [(abs(hash(w)) % (n - 1) + 1) for w in seq]
 
 
 class Tokenizer(object):
-    def __init__(self, nb_words=None, filters=base_filter(),
-                 lower=True, split=' ', char_level=False):
-        '''The class allows to vectorize a text corpus, by turning each
-        text into either a sequence of integers (each integer being the index
-        of a token in a dictionary) or into a vector where the coefficient
-        for each token could be binary, based on word count, based on tf-idf...
+    """Text tokenization utility class.
 
-        # Arguments
-            nb_words: the maximum number of words to keep, based
-                on word frequency. Only the most common `nb_words` words will
-                be kept.
-            filters: a string where each element is a character that will be
-                filtered from the texts. The default is all punctuation, plus
-                tabs and line breaks, minus the `'` character.
-            lower: boolean. Whether to convert the texts to lowercase.
-            split: character or string to use for token splitting.
-            char_level: if True, every character will be treated as a word.
+    This class allows to vectorize a text corpus, by turning each
+    text into either a sequence of integers (each integer being the index
+    of a token in a dictionary) or into a vector where the coefficient
+    for each token could be binary, based on word count, based on tf-idf...
 
-        By default, all punctuation is removed, turning the texts into
-        space-separated sequences of words
-        (words maybe include the `'` character). These sequences are then
-        split into lists of tokens. They will then be indexed or vectorized.
+    # Arguments
+        nb_words: the maximum number of words to keep, based
+            on word frequency. Only the most common `nb_words` words will
+            be kept.
+        filters: a string where each element is a character that will be
+            filtered from the texts. The default is all punctuation, plus
+            tabs and line breaks, minus the `'` character.
+        lower: boolean. Whether to convert the texts to lowercase.
+        split: character or string to use for token splitting.
+        char_level: if True, every character will be treated as a word.
 
-        `0` is a reserved index that won't be assigned to any word.
-        '''
+    By default, all punctuation is removed, turning the texts into
+    space-separated sequences of words
+    (words maybe include the `'` character). These sequences are then
+    split into lists of tokens. They will then be indexed or vectorized.
+
+    `0` is a reserved index that won't be assigned to any word.
+    """
+
+    def __init__(self, nb_words=None,
+                 filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
+                 lower=True,
+                 split=' ',
+                 char_level=False):
         self.word_counts = {}
         self.word_docs = {}
         self.filters = filters
@@ -75,16 +92,21 @@ class Tokenizer(object):
         self.char_level = char_level
 
     def fit_on_texts(self, texts):
-        '''Required before using texts_to_sequences or texts_to_matrix
+        """Updates internal vocabulary based on a list of texts.
+
+        Required before using `texts_to_sequences` or `texts_to_matrix`.
 
         # Arguments
             texts: can be a list of strings,
                 or a generator of strings (for memory-efficiency)
-        '''
+        """
         self.document_count = 0
         for text in texts:
             self.document_count += 1
-            seq = text if self.char_level else text_to_word_sequence(text, self.filters, self.lower, self.split)
+            seq = text if self.char_level else text_to_word_sequence(text,
+                                                                     self.filters,
+                                                                     self.lower,
+                                                                     self.split)
             for w in seq:
                 if w in self.word_counts:
                     self.word_counts[w] += 1
@@ -107,9 +129,15 @@ class Tokenizer(object):
             self.index_docs[self.word_index[w]] = c
 
     def fit_on_sequences(self, sequences):
-        '''Required before using sequences_to_matrix
-        (if fit_on_texts was never called)
-        '''
+        """Updates internal vocabulary based on a list of sequences.
+
+        Required before using `sequences_to_matrix`
+        (if `fit_on_texts` was never called).
+
+        # Arguments
+            sequences: A list of sequence.
+                A "sequence" is a list of integer word indices.
+        """
         self.document_count = len(sequences)
         self.index_docs = {}
         for seq in sequences:
@@ -121,30 +149,40 @@ class Tokenizer(object):
                     self.index_docs[i] += 1
 
     def texts_to_sequences(self, texts):
-        '''Transforms each text in texts in a sequence of integers.
+        """Transforms each text in texts in a sequence of integers.
+
         Only top "nb_words" most frequent words will be taken into account.
         Only words known by the tokenizer will be taken into account.
 
-        Returns a list of sequences.
-        '''
+        # Arguments
+            texts: A list of texts (strings).
+
+        # Returns
+            A list of sequences.
+        """
         res = []
         for vect in self.texts_to_sequences_generator(texts):
             res.append(vect)
         return res
 
     def texts_to_sequences_generator(self, texts):
-        '''Transforms each text in texts in a sequence of integers.
+        """Transforms each text in texts in a sequence of integers.
+
         Only top "nb_words" most frequent words will be taken into account.
         Only words known by the tokenizer will be taken into account.
 
-        Yields individual sequences.
+        # Arguments
+            texts: A list of texts (strings).
 
-        # Arguments:
-            texts: list of strings.
-        '''
+        # Yields
+            Yields individual sequences.
+        """
         nb_words = self.nb_words
         for text in texts:
-            seq = text if self.char_level else text_to_word_sequence(text, self.filters, self.lower, self.split)
+            seq = text if self.char_level else text_to_word_sequence(text,
+                                                                     self.filters,
+                                                                     self.lower,
+                                                                     self.split)
             vect = []
             for w in seq:
                 i = self.word_index.get(w)
@@ -156,39 +194,47 @@ class Tokenizer(object):
             yield vect
 
     def texts_to_matrix(self, texts, mode='binary'):
-        '''Convert a list of texts to a Numpy matrix,
-        according to some vectorization mode.
+        """Convert a list of texts to a Numpy matrix.
 
-        # Arguments:
+        # Arguments
             texts: list of strings.
-            modes: one of "binary", "count", "tfidf", "freq"
-        '''
+            mode: one of "binary", "count", "tfidf", "freq".
+
+        # Returns
+            A Numpy matrix.
+        """
         sequences = self.texts_to_sequences(texts)
         return self.sequences_to_matrix(sequences, mode=mode)
 
     def sequences_to_matrix(self, sequences, mode='binary'):
-        '''Converts a list of sequences into a Numpy matrix,
-        according to some vectorization mode.
+        """Converts a list of sequences into a Numpy matrix.
 
-        # Arguments:
+        # Arguments
             sequences: list of sequences
                 (a sequence is a list of integer word indices).
-            modes: one of "binary", "count", "tfidf", "freq"
-        '''
+            mode: one of "binary", "count", "tfidf", "freq"
+
+        # Returns
+            A Numpy matrix.
+
+        # Raises
+            ValueError: In case of invalid `mode` argument,
+                or if the Tokenizer requires to be fit to sample data.
+        """
         if not self.nb_words:
             if self.word_index:
                 nb_words = len(self.word_index) + 1
             else:
-                raise Exception('Specify a dimension (nb_words argument), '
-                                'or fit on some text data first.')
+                raise ValueError('Specify a dimension (nb_words argument), '
+                                 'or fit on some text data first.')
         else:
             nb_words = self.nb_words
 
         if mode == 'tfidf' and not self.document_count:
-            raise Exception('Fit the Tokenizer on some data '
-                            'before using tfidf mode.')
+            raise ValueError('Fit the Tokenizer on some data '
+                             'before using tfidf mode.')
 
-        X = np.zeros((len(sequences), nb_words))
+        x = np.zeros((len(sequences), nb_words))
         for i, seq in enumerate(sequences):
             if not seq:
                 continue
@@ -202,17 +248,18 @@ class Tokenizer(object):
                     counts[j] += 1
             for j, c in list(counts.items()):
                 if mode == 'count':
-                    X[i][j] = c
+                    x[i][j] = c
                 elif mode == 'freq':
-                    X[i][j] = c / len(seq)
+                    x[i][j] = c / len(seq)
                 elif mode == 'binary':
-                    X[i][j] = 1
+                    x[i][j] = 1
                 elif mode == 'tfidf':
                     # Use weighting scheme 2 in
-                    #   https://en.wikipedia.org/wiki/Tf%E2%80%93idf
+                    # https://en.wikipedia.org/wiki/Tf%E2%80%93idf
                     tf = 1 + np.log(c)
-                    idf = np.log(1 + self.document_count / (1 + self.index_docs.get(j, 0)))
-                    X[i][j] = tf * idf
+                    idf = np.log(1 + self.document_count /
+                                 (1 + self.index_docs.get(j, 0)))
+                    x[i][j] = tf * idf
                 else:
-                    raise Exception('Unknown vectorization mode: ' + str(mode))
-        return X
+                    raise ValueError('Unknown vectorization mode:', mode)
+        return x

keras/regularizers.py

@@ -1,98 +1,84 @@
 from __future__ import absolute_import
 from . import backend as K
 from .utils.generic_utils import get_from_module
+import warnings
 
 
 class Regularizer(object):
+    """Regularizer base class.
+    """
 
-    def set_param(self, p):
-        self.p = p
-
-    def set_layer(self, layer):
-        self.layer = layer
-
-    def __call__(self, loss):
-        return loss
+    def __call__(self, x):
+        return 0
 
     def get_config(self):
         return {'name': self.__class__.__name__}
 
+    def set_param(self, _):
+        warnings.warn('The `set_param` method on regularizers is deprecated. '
+                      'It no longer does anything, '
+                      'and it will be removed after 06/2017.')
+
+    def set_layer(self, _):
+        warnings.warn('The `set_layer` method on regularizers is deprecated. '
+                      'It no longer does anything, '
+                      'and it will be removed after 06/2017.')
+
 
 class EigenvalueRegularizer(Regularizer):
-    '''This takes a constant that controls
-    the regularization by Eigenvalue Decay on the
-    current layer and outputs the regularized
-    loss (evaluated on the training data) and
-    the original loss (evaluated on the
-    validation data).
-    '''
+    """Regularizer based on the eignvalues of a weight matrix.
+
+    Only available for tensors of rank 2.
+
+    # Arguments
+        k: Float; modulates the amount of regularization to apply.
+    """
+
     def __init__(self, k):
         self.k = k
-        self.uses_learning_phase = True
 
-    def set_param(self, p):
-        if hasattr(self, 'p'):
-            raise Exception('Regularizers cannot be reused. '
-                            'Instantiate one regularizer per layer.')
-        self.p = p
+    def __call__(self, x):
+        if K.ndim(x) != 2:
+            raise ValueError('EigenvalueRegularizer '
+                             'is only available for tensors of rank 2.')
+        covariance = K.dot(K.transpose(x), x)
+        dim1, dim2 = K.eval(K.shape(covariance))
 
-    def __call__(self, loss):
-        power = 9  # number of iterations of the power method
-        W = self.p
-        if K.ndim(W) > 2:
-            raise Exception('Eigenvalue Decay regularizer '
-                            'is only available for dense '
-                            'and embedding layers.')
-        WW = K.dot(K.transpose(W), W)
-        dim1, dim2 = K.eval(K.shape(WW))  # number of neurons in the layer
-
-        # power method for approximating the dominant eigenvector:
-        o = K.ones([dim1, 1])  # initial values for the dominant eigenvector
-        main_eigenvect = K.dot(WW, o)
+        # Power method for approximating the dominant eigenvector:
+        power = 9  # Number of iterations of the power method.
+        o = K.ones([dim1, 1])  # Initial values for the dominant eigenvector.
+        main_eigenvect = K.dot(covariance, o)
         for n in range(power - 1):
-            main_eigenvect = K.dot(WW, main_eigenvect)
+            main_eigenvect = K.dot(covariance, main_eigenvect)
+        covariance_d = K.dot(covariance, main_eigenvect)
 
-        WWd = K.dot(WW, main_eigenvect)
-
-        # the corresponding dominant eigenvalue:
-        main_eigenval = (K.dot(K.transpose(WWd), main_eigenvect) /
+        # The corresponding dominant eigenvalue:
+        main_eigenval = (K.dot(K.transpose(covariance_d), main_eigenvect) /
                          K.dot(K.transpose(main_eigenvect), main_eigenvect))
-        # multiplied by the given regularization gain
-        regularized_loss = loss + (main_eigenval ** 0.5) * self.k
-
-        return K.in_train_phase(regularized_loss[0, 0], loss)
+        # Multiply by the given regularization gain.
+        regularization = (main_eigenval ** 0.5) * self.k
+        return K.sum(regularization)
 
 
-class WeightRegularizer(Regularizer):
+class L1L2Regularizer(Regularizer):
+    """Regularizer for L1 and L2 regularization.
+
+    # Arguments
+        l1: Float; L1 regularization factor.
+        l2: Float; L2 regularization factor.
+    """
 
     def __init__(self, l1=0., l2=0.):
         self.l1 = K.cast_to_floatx(l1)
         self.l2 = K.cast_to_floatx(l2)
-        self.uses_learning_phase = True
-        self.p = None
 
-    def set_param(self, p):
-        if self.p is not None:
-            raise Exception('Regularizers cannot be reused. '
-                            'Instantiate one regularizer per layer.')
-        self.p = p
-
-    def __call__(self, loss):
-        if self.p is None:
-            raise Exception('Need to call `set_param` on '
-                            'WeightRegularizer instance '
-                            'before calling the instance. '
-                            'Check that you are not passing '
-                            'a WeightRegularizer instead of an '
-                            'ActivityRegularizer '
-                            '(i.e. activity_regularizer="l2" instead '
-                            'of activity_regularizer="activity_l2".')
-        regularized_loss = loss
+    def __call__(self, x):
+        regularization = 0
         if self.l1:
-            regularized_loss += K.sum(self.l1 * K.abs(self.p))
+            regularization += K.sum(self.l1 * K.abs(x))
         if self.l2:
-            regularized_loss += K.sum(self.l2 * K.square(self.p))
-        return K.in_train_phase(regularized_loss, loss)
+            regularization += K.sum(self.l2 * K.square(x))
+        return regularization
 
     def get_config(self):
         return {'name': self.__class__.__name__,
@@ -100,61 +86,34 @@ class WeightRegularizer(Regularizer):
                 'l2': float(self.l2)}
 
 
-class ActivityRegularizer(Regularizer):
+# Aliases.
 
-    def __init__(self, l1=0., l2=0.):
-        self.l1 = K.cast_to_floatx(l1)
-        self.l2 = K.cast_to_floatx(l2)
-        self.uses_learning_phase = True
-        self.layer = None
-
-    def set_layer(self, layer):
-        if self.layer is not None:
-            raise Exception('Regularizers cannot be reused')
-        self.layer = layer
-
-    def __call__(self, loss):
-        if self.layer is None:
-            raise Exception('Need to call `set_layer` on '
-                            'ActivityRegularizer instance '
-                            'before calling the instance.')
-        regularized_loss = loss
-        for i in range(len(self.layer.inbound_nodes)):
-            output = self.layer.get_output_at(i)
-            if self.l1:
-                regularized_loss += K.sum(self.l1 * K.abs(output))
-            if self.l2:
-                regularized_loss += K.sum(self.l2 * K.square(output))
-        return K.in_train_phase(regularized_loss, loss)
-
-    def get_config(self):
-        return {'name': self.__class__.__name__,
-                'l1': float(self.l1),
-                'l2': float(self.l2)}
+WeightRegularizer = L1L2Regularizer
+ActivityRegularizer = L1L2Regularizer
 
 
 def l1(l=0.01):
-    return WeightRegularizer(l1=l)
+    return L1L2Regularizer(l1=l)
 
 
 def l2(l=0.01):
-    return WeightRegularizer(l2=l)
+    return L1L2Regularizer(l2=l)
 
 
 def l1l2(l1=0.01, l2=0.01):
-    return WeightRegularizer(l1=l1, l2=l2)
+    return L1L2Regularizer(l1=l1, l2=l2)
 
 
 def activity_l1(l=0.01):
-    return ActivityRegularizer(l1=l)
+    return L1L2Regularizer(l1=l)
 
 
 def activity_l2(l=0.01):
-    return ActivityRegularizer(l2=l)
+    return L1L2Regularizer(l2=l)
 
 
 def activity_l1l2(l1=0.01, l2=0.01):
-    return ActivityRegularizer(l1=l1, l2=l2)
+    return L1L2Regularizer(l1=l1, l2=l2)
 
 
 def get(identifier, kwargs=None):

keras/utils/data_utils.py

@@ -1,21 +1,38 @@
+"""Utilities for file download and caching."""
 from __future__ import absolute_import
 from __future__ import print_function
 
+import functools
 import tarfile
 import os
 import sys
 import shutil
 import hashlib
 from six.moves.urllib.request import urlopen
-from six.moves.urllib.error import URLError, HTTPError
+from six.moves.urllib.error import URLError
+from six.moves.urllib.error import HTTPError
 
 from ..utils.generic_utils import Progbar
 
 
-# Under Python 2, 'urlretrieve' relies on FancyURLopener from legacy
-# urllib module, known to have issues with proxy management
 if sys.version_info[0] == 2:
     def urlretrieve(url, filename, reporthook=None, data=None):
+        """Replacement for `urlretrive` for Python 2.
+
+        Under Python 2, `urlretrieve` relies on `FancyURLopener` from legacy
+        `urllib` module, known to have issues with proxy management.
+
+        # Arguments
+            url: url to retrieve.
+            filename: where to store the retrieved data locally.
+            reporthook: a hook function that will be called once
+                on establishment of the network connection and once
+                after each block read thereafter.
+                The hook will be passed three arguments;
+                a count of blocks transferred so far,
+                a block size in bytes, and the total size of the file.
+            data: `data` argument passed to `urlopen`.
+        """
         def chunk_read(response, chunk_size=8192, reporthook=None):
             total_size = response.info().get('Content-Length').strip()
             total_size = int(total_size)
@@ -40,6 +57,21 @@ else:
 
 def get_file(fname, origin, untar=False,
              md5_hash=None, cache_subdir='datasets'):
+    """Downloads a file from a URL if it not already in the cache.
+
+    Passing the MD5 hash will verify the file after download
+    as well as if it is already present in the cache.
+
+    # Arguments
+        fname: name of the file
+        origin: original URL of the file
+        untar: boolean, whether the file should be decompressed
+        md5_hash: MD5 hash of the file for verification
+        cache_subdir: directory being used as the cache
+
+    # Returns
+        Path to the downloaded file
+    """
     datadir_base = os.path.expanduser(os.path.join('~', '.keras'))
     if not os.access(datadir_base, os.W_OK):
         datadir_base = os.path.join('/tmp', '.keras')
@@ -55,7 +87,7 @@ def get_file(fname, origin, untar=False,
 
     download = False
     if os.path.exists(fpath):
-        # file found; verify integrity if a hash was provided
+        # File found; verify integrity if a hash was provided.
         if md5_hash is not None:
             if not validate_file(fpath, md5_hash):
                 print('A local file was found, but it seems to be '
@@ -66,11 +98,9 @@ def get_file(fname, origin, untar=False,
 
     if download:
         print('Downloading data from', origin)
-        global progbar
         progbar = None
 
-        def dl_progress(count, block_size, total_size):
-            global progbar
+        def dl_progress(count, block_size, total_size, progbar=None):
             if progbar is None:
                 progbar = Progbar(total_size)
             else:
@@ -79,7 +109,8 @@ def get_file(fname, origin, untar=False,
         error_msg = 'URL fetch failure on {}: {} -- {}'
         try:
             try:
-                urlretrieve(origin, fpath, dl_progress)
+                urlretrieve(origin, fpath,
+                            functools.partial(dl_progress, progbar=progbar))
             except URLError as e:
                 raise Exception(error_msg.format(origin, e.errno, e.reason))
             except HTTPError as e:
@@ -110,6 +141,15 @@ def get_file(fname, origin, untar=False,
 
 
 def validate_file(fpath, md5_hash):
+    """Validates a file against a MD5 hash.
+
+    # Arguments
+        fpath: path to the file being validated
+        md5_hash: the MD5 hash being validated against
+
+    # Returns
+        Whether the file is valid
+    """
     hasher = hashlib.md5()
     with open(fpath, 'rb') as f:
         buf = f.read()